Docusign Monitor safeguards agreements with round-the-clock activity tracking

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IdentityModel.Tokens;
using System.IdentityModel.Tokens.Jwt;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Runtime.Caching;
using System.Security.Claims;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Org.BouncyCastle.Crypto;
using Org.BouncyCastle.Crypto.Parameters;
using Org.BouncyCastle.OpenSsl;
using Org.BouncyCastle.Security;

namespace DocuSignMonitorEventDownloader
{
    public class Program
    {
        #region These will change based on your settings
        private static readonly string IntegratorKey = "Your Integrator Key Here";
        private static readonly Guid UserId = Guid.Parse("The UserId GUID of the user we're impersonating");
        private static readonly string SecretKeyLocation = "PathToYourSecretKey";
        private static readonly string DownloadLocation = "C:\\DocuSignMonitor\\download";
        #endregion
        
        // Demo environment connection parameters
        private static readonly string OauthUrl = "https://account-d.docusign.com/oauth/token";
        private static readonly string Aud = "account-d.docusign.com";
        private static readonly string Host = "lens-d.docusign.net";
        
        // API connection details
        private static readonly string DownloadEventsPath = "/api/v2.0/datasets/monitor/stream";
        private static readonly string Scheme = "https";
        private static readonly int Port = 443;
        private static readonly int MaxRetryCount = 3;

        private static readonly MemoryCache Cache = new MemoryCache("DocuSignMonitorEventDownloader");
        
        public static void Main(string[] args)
        {
            try
            {
                DownloadAllDataAsync().Wait();
            }
            catch (AggregateException ae)
            {
                Console.Out.WriteLine($"Aggregate Exception, {ae.InnerExceptions.Count} inner exceptions");
                foreach (Exception e in ae.Flatten().InnerExceptions)
                {
                    Console.Out.WriteLine(e);
                }
            }

            Console.Out.WriteLine("Press any key to continue...");
            Console.ReadKey();
        }

        /// <summary>
        /// Auth, then connect to our download endpoint and continue to request data until we are caught up.
        /// Once caught up, we're done for our simple example.
        /// </summary>
        public static async Task DownloadAllDataAsync()
        {
            string dir = Path.Combine(DownloadLocation, IntegratorKey);
            Directory.CreateDirectory(dir);
            string path = Path.Combine(dir, "data.ndjson");
            File.Delete(path);
            Console.Out.WriteLine("Writing data to: " + path);
            
            long totalCount = 0;
            string cursor = string.Empty;//"aa_0_0_0";
            Stopwatch total = Stopwatch.StartNew();

            while (true) // while there might be data to fetch, fetch data until we run out
            {
                // get our data, up to ~500 rows at a time
                Stopwatch batchTimer = Stopwatch.StartNew();
                EventsResult eventsResult = await GetEvents(GetAuthorizedClient(), cursor, 1000);
                
                // log
                Console.Out.WriteLine($"[{cursor}->{eventsResult.endCursor}]  RowCount: {eventsResult.data.Length} in {batchTimer.Elapsed.TotalSeconds:0}s");
                
                // save data to disk
                if (eventsResult.data.Length > 0)
                {
                    string resultsAsNdJson = string.Join("\n", eventsResult.data.Select(x => JsonConvert.SerializeObject(x, Formatting.None)));
                    File.AppendAllText(path, resultsAsNdJson, Encoding.UTF8);
                }
                
                // see if our cursor advanced since last pull (new data), if not we're done in this simple example
                else if(cursor == eventsResult.endCursor) 
                {
                    break; // caught up, we're done
                }
                
                totalCount += eventsResult.data.Length;
                cursor = eventsResult.endCursor; // end becomes start for next iteration
                Thread.Sleep(3000); // don't hammer
            }
     
            Console.Out.WriteLine($"Complete, downloaded {totalCount:N0} events in {(int)total.Elapsed.TotalSeconds:N0} seconds");
        }
        
        private static async Task<T> GetFromUrl<T>(HttpClient client, UriBuilder builder)
        {
            int attempt = 0;

            // in case we get throttled via HTTP 429, wait a bit, and try again
            while (attempt < MaxRetryCount)
            {
                if (attempt > 0)
                {
                    Thread.Sleep(TimeSpan.FromSeconds(20));
                }

                attempt++;
                
                HttpResponseMessage response;
                try
                {
                    response = client.GetAsync(builder.Uri).Result;
                }
                catch (Exception e)
                {
                    throw new Exception($"Error connecting to  {builder.Uri}", e);
                }

                if (response.IsSuccessStatusCode)
                {
                    return await response.Content.ReadAsAsync<T>();
                }
                else if (response.StatusCode == HttpStatusCode.NotFound)
                {
                    throw new Exception($"The URL:  {builder.Uri}  could not be located (404)");
                }
                else if ((int)response.StatusCode != 429)
                {
                    //response.Headers.GetValues("X-Docusign-TraceToken") ?? new []{"NO TRACE TOKEN"}
                    throw new Exception($"StatusCode: '{(int)response.StatusCode}' Reason: '{response.ReasonPhrase}' URL: '{builder.Uri}' Content:  {await response.Content.ReadAsStringAsync()}");
                }

                if (attempt == 1)
                {
                    Console.Out.WriteLine($"429: {builder.Uri} is throttled, attempting up to {MaxRetryCount} retries");
                }
            }

            throw new Exception($"Exceeded max retry calling: '{builder.Uri}'");
        }
        
        private static async Task<EventsResult> GetEvents(HttpClient client, string cursor, int limit)
        {
            UriBuilder builder = new UriBuilder
            {
                Scheme = Scheme,
                Host = Host,
                Port = Port,
                Path = DownloadEventsPath,
                Query = $"cursor={cursor}&limit={limit}"
            };

            var ret = await GetFromUrl<JObject>(client, builder);
            return ret.ToObject<EventsResult>();
        }

        //OAuth.  JWT must be for an integrator token.  
        static HttpClient GetAuthorizedClient()
        {
            string cacheKey = "HttpClient";
            HttpClient ret = (HttpClient) Cache.Get(cacheKey);
            if (ret != null)
            {
                return ret;
            }

            int expireInMinutes = 10;
            string assertion = GenerateJwt(expireInMinutes);

            OAuth oAuth;

            HttpClient oAuthClient = new HttpClient();
            oAuthClient.DefaultRequestHeaders.Accept.Clear();
            oAuthClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));

            var body = new List<KeyValuePair<string, string>>
            {
                new KeyValuePair<string, string>("grant_type", "urn:ietf:params:oauth:grant-type:jwt-bearer"),
                new KeyValuePair<string, string>("assertion", assertion)
            };

            HttpResponseMessage response = oAuthClient.PostAsync(OauthUrl, new FormUrlEncodedContent(body)).Result;
            if (response.IsSuccessStatusCode)
            {
                oAuth = response.Content.ReadAsAsync<OAuth>().Result;
            }
            else
            {
                // if you have to speak with a Docusign customer support rep, the TraceToken value is helpful in locating your error 
                if (!response.Headers.TryGetValues("X-Docusign-TraceToken", out var traceTokenValues))
                {
                    traceTokenValues = new[] {"NO TRACE TOKEN"};
                }
                string traceToken = string.Join(", ", traceTokenValues);
                throw new Exception("Failed authentication, reason: '" + response.ReasonPhrase + "' TraceToken: '"+traceToken+"' data, if any: " + response.Content.ReadAsStringAsync().Result);
            }

            ret = new HttpClient(new HttpClientHandler
            {
                AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
            });
            ret.DefaultRequestHeaders.Accept.Clear();
            ret.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
            ret.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/x-ndjson"));
            ret.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("bearer", oAuth.access_token);
            ret.DefaultRequestHeaders.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip"));

            // pulling the data can take a long time when starting from the beginning
            ret.Timeout = TimeSpan.FromMinutes(5);

            Console.Out.WriteLine($"Successfully authorized for {expireInMinutes} minutes from {OauthUrl} for IK: {IntegratorKey} impersonating user: {UserId}");
            
            Cache.Set(cacheKey, ret, DateTimeOffset.UtcNow.AddMinutes(expireInMinutes/2));
            
            return ret;
        }

        private static string GenerateJwt(int expireMinutes = 20)
        {
            RSA rsa = CreateRsaKeyFromPem(GetSecretKey());
            var rsaKey = new Microsoft.IdentityModel.Tokens.RsaSecurityKey(rsa);
            var creds = new Microsoft.IdentityModel.Tokens.SigningCredentials(rsaKey, SecurityAlgorithms.RsaSha256Signature, SecurityAlgorithms.HmacSha256Signature);

            ClaimsIdentity claims = new ClaimsIdentity(new []
            {
                new Claim("scope", "signature impersonation"),
                new Claim("sub", UserId.ToString())
            });

            JwtSecurityTokenHandler handler = new JwtSecurityTokenHandler {SetDefaultTimesOnTokenCreation = false};
            var token = handler.CreateJwtSecurityToken(IntegratorKey, Aud, claims, null, DateTime.UtcNow.AddMinutes(expireMinutes), DateTime.UtcNow, creds);
            
            string jwtToken = handler.WriteToken(token);

            return jwtToken;
        }

        private static string GetSecretKey()
        {
            if (File.Exists(SecretKeyLocation))
            {
                return File.ReadAllText(SecretKeyLocation);
            }
            else
            {
                throw new FileNotFoundException($"Unable to locate the specified secret file at location: '{SecretKeyLocation}'");
            }
        }

        private static RSA CreateRsaKeyFromPem(string key)
        {
            TextReader reader = new StringReader(key);
            PemReader pemReader = new PemReader(reader);

            object result = pemReader.ReadObject();

            if (result is AsymmetricCipherKeyPair keyPair)
            {
                return DotNetUtilities.ToRSA((RsaPrivateCrtKeyParameters)keyPair.Private);
            }
            else if (result is RsaKeyParameters keyParameters)
            {
                return DotNetUtilities.ToRSA(keyParameters);
            }

            throw new Exception("Unepxected PEM type");
        }

        public class EventsResult
        {
            public string endCursor { get; set; }

            public Event[] data { get; set; }
        }
        
        public class Event
        {
            public DateTime timestamp { get; set; } // UTC

            public string eventId { get; set; }

            public string site { get; set; }

            public string accountId { get; set; }
            
            public string organizationId { get; set; }
     
            public string userId { get; set; }
     
            public string integratorKey { get; set; }
        
            public string userAgent { get; set; }
            
            public UserAgentClientInfo UserAgentClientInfo { get; set; }

            public string ipAddress { get; set; }
            
            public IpAddressLocation ipAddressLocation { get; set; }

            public string @object { get; set; }

            public string action { get; set; }
  
            public string property { get; set; }

            public string field { get; set; }

            public string result { get; set; }

             public JObject data { get; set; }
        } 
        
        public class UserAgentClientInfo
        {
            
            [JsonProperty("browser")]
            public Browser Browser { get; set; }
                
            [JsonProperty("device")]
            public Device Device { get; set; }
                
            [JsonProperty("os")]
            public Os Os { get; set; }

            public override string ToString() => $"Browser: {Browser}\tDevice: {Device}\tOs: {Os}";
        }
        
        public class Browser
        {
            [JsonProperty("family")]
            public string Family { get; set; }
                
            [JsonProperty("version")]
            public Version Version { get; set; }
            
            public override string ToString() => $"{Family} {Version}";
        }

        public class Device
        {
            [JsonProperty("family")]
            public string Family { get; set; }
                
            [JsonProperty("brand")]
            public string Brand { get; set; }
                
            [JsonProperty("model")]
            public string Model { get; set; }

            public override string ToString() => $"{Family} {Brand} {Model}";
        }

        public class Os
        {
            [JsonProperty("family")]
            public string Family { get; set; }
                
            [JsonProperty("version")]
            public Version Version { get; set; }
            
            public override string ToString() => $"{Family} {Version}";
        }

        public class Version
        {
            [JsonProperty("major")]
            public string Major { get; set; }
                
            [JsonProperty("minor")]
            public string Minor { get; set; }
                
            [JsonProperty("patch")]
            public string Patch { get; set; }

            public override string ToString() => $"{Major}.{Minor}.{Patch}";
        }
        
        public class IpAddressLocation
        {
            [JsonProperty("latitude")]
            public double Latitude { get; set; }
            
            [JsonProperty("longitude")]
            public double Longitude { get; set; }
            
            [JsonProperty("country")]
            public string Country { get; set; }
            
            [JsonProperty("state")]
            public string State { get; set; }
            
            [JsonProperty("city")]
            public string City { get; set; }

            public override string ToString() => $"{Country}.{State}.{City}";
        }

        public class OAuth
        {
            public string access_token { get; set; }
            public string token_type { get; set; }
            public string expires_in { get; set; }
        }
    }
}