-
Notifications
You must be signed in to change notification settings - Fork 0
/
DownloadService.cs
203 lines (182 loc) · 8.67 KB
/
DownloadService.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
namespace DelugeSync
{
public static class DownloadService
{
public static ILogger _logger;
public static int IdleConnectionSeconds = 10;
public static int DownloadChunks = 4;
public static int MaxConnections = 8;
public static bool NagleAlgorithm = true; //enabled by default
//beta options
public static bool BetaOptions = false;
public static bool UnsafeAuthenticatedConnectionSharing = true;
public static bool PreAuthenticate = true;
public static bool AllowWriteStreamBuffering = false;
public static bool Pipelined = false;
static DownloadService()
{
ServicePointManager.Expect100Continue = false;
ServicePointManager.UseNagleAlgorithm = NagleAlgorithm;
ServicePointManager.DefaultConnectionLimit = MaxConnections;
ServicePointManager.MaxServicePointIdleTime = (IdleConnectionSeconds * 1000);
//ServicePointManager.ReusePort = false;
}
public static async Task<DownloadResult> DownloadAsync(string fileUrl, string destinationFolderPath, int numberOfParallelDownloads = 0, bool validateSSL = false, NetworkCredential credentials = null, string tempFolderPath = null)
{
try
{
if (!validateSSL)
{
_logger.LogInformation("Validating SSL");
ServicePointManager.ServerCertificateValidationCallback = delegate { return true; };
}
Uri uri = new Uri(fileUrl);
string destinationFilePath;
if (!string.IsNullOrEmpty(tempFolderPath))
{
destinationFilePath = tempFolderPath;
} else
{
destinationFilePath = destinationFolderPath;
}
DownloadResult result = new DownloadResult() { FilePath = destinationFilePath };
//Basically parallel downloads must = chunks
if (numberOfParallelDownloads <= 0)
{
numberOfParallelDownloads = DownloadChunks;
}
_logger.LogInformation("Requesting file information from server");
#region Get file size
WebRequest webRequest = HttpWebRequest.Create(fileUrl);
webRequest.Method = "HEAD";
if (credentials != null) webRequest.Credentials = credentials;
long responseLength;
using (WebResponse webResponse = webRequest.GetResponse())
{
responseLength = long.Parse(webResponse.Headers.Get("Content-Length"));
result.Size = responseLength;
}
#endregion
if (File.Exists(destinationFilePath))
{
File.Delete(destinationFilePath);
}
using (FileStream destinationStream = new FileStream(destinationFilePath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite))
{
ConcurrentDictionary<long, string> tempFilesDictionary = new ConcurrentDictionary<long, string>();
_logger.LogInformation("Calculating ranges");
#region Calculate ranges
List<Range> readRanges = new List<Range>();
for (int chunk = 0; chunk < numberOfParallelDownloads - 1; chunk++)
{
var range = new Range()
{
Start = chunk * (responseLength / numberOfParallelDownloads),
End = ((chunk + 1) * (responseLength / numberOfParallelDownloads)) - 1
};
readRanges.Add(range);
}
readRanges.Add(new Range()
{
Start = readRanges.Any() ? readRanges.Last().End + 1 : 0,
End = responseLength
});
#endregion
DateTime startTime = DateTime.Now;
_logger.LogInformation("Starting download...");
#region Parallel download
int index = 0;
await Parallel.ForEachAsync(readRanges, new ParallelOptions() { MaxDegreeOfParallelism = numberOfParallelDownloads }, async (readRange, cancellationToken) =>
{
try
{
HttpWebRequest httpWebRequest = HttpWebRequest.Create(fileUrl) as HttpWebRequest;
httpWebRequest.Method = "GET";
httpWebRequest.Proxy = null;
if (BetaOptions)
{
httpWebRequest.UnsafeAuthenticatedConnectionSharing = UnsafeAuthenticatedConnectionSharing;
httpWebRequest.PreAuthenticate = PreAuthenticate;
httpWebRequest.AllowWriteStreamBuffering = AllowWriteStreamBuffering;
httpWebRequest.Pipelined = Pipelined;
}
if (credentials != null)
{
httpWebRequest.Credentials = credentials;
}
httpWebRequest.AddRange(readRange.Start, readRange.End);
using (HttpWebResponse httpWebResponse = await httpWebRequest.GetResponseAsync() as HttpWebResponse)
{
//TODO: Create own temp file config
string tempFilePath = Path.GetTempFileName();
using (var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite))
{
await httpWebResponse.GetResponseStream().CopyToAsync(fileStream);
var result = tempFilesDictionary.TryAdd(readRange.Start, tempFilePath);
if (!result) _logger.LogError("Key already exists");
}
}
} catch (Exception ex)
{
_logger.LogError(ex.ToString());
}
index++;
});
result.ParallelDownloads = index;
#endregion
result.TimeTaken = DateTime.Now.Subtract(startTime);
_logger.LogInformation("Merging chunks");
#region Merge to single file
foreach (var tempFile in tempFilesDictionary.OrderBy(b => b.Key))
{
byte[] tempFileBytes = File.ReadAllBytes(tempFile.Value);
destinationStream.Write(tempFileBytes, 0, tempFileBytes.Length);
File.Delete(tempFile.Value);
}
#endregion
_logger.LogInformation("Moving file from temporary location");
#region move to real location if temp file location is specified
if (!string.IsNullOrEmpty(tempFolderPath))
{
try
{
File.Move(destinationFilePath, destinationFolderPath, true);
} catch (Exception ex)
{
_logger.LogError($"Failed to move file: {ex.ToString()}");
}
}
#endregion
await destinationStream.FlushAsync();
await destinationStream.DisposeAsync();
return result;
}
} catch (Exception ex)
{
_logger.LogError(ex.ToString());
return null;
}
}
}
internal class Range
{
public long Start { get; set; }
public long End { get; set; }
}
public class DownloadResult
{
public long Size { get; set; }
public string FilePath { get; set; }
public TimeSpan TimeTaken { get; set; }
public int ParallelDownloads { get; set; }
}
}