diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..8ca517d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "nuget" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" diff --git a/CosmosClone/CloneConsoleRun/App.config b/CosmosClone/CloneConsoleRun/App.config index 0ee77ab..c484c1b 100644 --- a/CosmosClone/CloneConsoleRun/App.config +++ b/CosmosClone/CloneConsoleRun/App.config @@ -1,74 +1,58 @@ - + -
-
-
+
+
+
- - - - - + + + + + - - - - - + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - \ No newline at end of file + diff --git a/CosmosClone/CloneConsoleRun/CloneConsoleRun.csproj b/CosmosClone/CloneConsoleRun/CloneConsoleRun.csproj index 4314182..7c31e25 100644 --- a/CosmosClone/CloneConsoleRun/CloneConsoleRun.csproj +++ b/CosmosClone/CloneConsoleRun/CloneConsoleRun.csproj @@ -8,11 +8,12 @@ Exe CloneConsoleRun CloneConsoleRun - v4.6.1 + v4.8 512 true + AnyCPU @@ -34,42 +35,8 @@ 4 - - ..\packages\Microsoft.Azure.CosmosDB.BulkExecutor.1.2.0\lib\net451\Microsoft.Azure.CosmosDB.BulkImport.dll - - - ..\packages\Microsoft.Azure.DocumentDB.2.1.3\lib\net461\Microsoft.Azure.Documents.Client.dll - - - ..\packages\MongoDB.Bson.2.4.4\lib\net45\MongoDB.Bson.dll - - - ..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll - - - ..\packages\System.Buffers.4.5.0\lib\netstandard2.0\System.Buffers.dll - - - ..\packages\System.Memory.4.5.0\lib\netstandard2.0\System.Memory.dll - - - ..\packages\System.Net.Http.WinHttpHandler.4.5.0\lib\net461\System.Net.Http.WinHttpHandler.dll - - - - ..\packages\System.Numerics.Vectors.4.4.0\lib\net46\System.Numerics.Vectors.dll - - - ..\packages\System.Runtime.CompilerServices.Unsafe.4.5.0\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll - - - - - - - @@ -81,7 +48,6 @@ - @@ -89,12 +55,22 @@ CosmosCloneCommon + + + 1.8.9 + + + 2.24.0 + + + 2.19.0 + + + 13.0.3 + + + 6.0.0 + + - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - \ No newline at end of file diff --git a/CosmosClone/CloneConsoleRun/Program.cs b/CosmosClone/CloneConsoleRun/Program.cs index d989ca9..cb94e9a 100644 --- a/CosmosClone/CloneConsoleRun/Program.cs +++ b/CosmosClone/CloneConsoleRun/Program.cs @@ -4,15 +4,11 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using CosmosCloneCommon; + using CosmosCloneCommon.Migrator; using CosmosCloneCommon.Model; -using CosmosCloneCommon.Utility; + using logger = CosmosCloneCommon.Utility.CloneLogger; -using Newtonsoft.Json; namespace CloneConsoleRun { @@ -72,8 +68,8 @@ public static void TestCosmosScrubbing() var documentMigrator = new CosmosCloneCommon.Migrator.DocumentMigrator(); documentMigrator.StartCopy(scrubRules).Wait(); //var result = tcs.StartScrub(scrubRules); - } + } + - } } diff --git a/CosmosClone/CloneConsoleRun/Properties/AssemblyInfo.cs b/CosmosClone/CloneConsoleRun/Properties/AssemblyInfo.cs index 37001c1..3751390 100644 --- a/CosmosClone/CloneConsoleRun/Properties/AssemblyInfo.cs +++ b/CosmosClone/CloneConsoleRun/Properties/AssemblyInfo.cs @@ -1,5 +1,4 @@ using System.Reflection; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; // General Information about an assembly is controlled through the following diff --git a/CosmosClone/CloneConsoleRun/Sample/CosmosSampleDBHelper.cs b/CosmosClone/CloneConsoleRun/Sample/CosmosSampleDBHelper.cs index d2cad27..314e0aa 100644 --- a/CosmosClone/CloneConsoleRun/Sample/CosmosSampleDBHelper.cs +++ b/CosmosClone/CloneConsoleRun/Sample/CosmosSampleDBHelper.cs @@ -7,12 +7,14 @@ namespace CloneConsoleRun.Sample { using System; using System.Threading.Tasks; + using CosmosCloneCommon.Utility; + using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; public class CosmosSampleDBHelper - { + { private ConnectionPolicy ConnectionPolicy; public CosmosSampleDBHelper() { @@ -25,7 +27,7 @@ public CosmosSampleDBHelper() this.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 20; this.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 600; } - + public DocumentClient GetSampleDocumentDbClient() { try @@ -68,7 +70,7 @@ public async Task CreateSampleDocumentCollection(DocumentCli //no partition key if it is a fixed collection newDocumentCollection = (DocumentCollection)await sampleClient.CreateDocumentCollectionIfNotExistsAsync (UriFactory.CreateDatabaseUri(sampleDatabaseName), - new DocumentCollection { Id = sampleCollectionName}, + new DocumentCollection { Id = sampleCollectionName }, new RequestOptions { OfferThroughput = offerThroughput }); } return newDocumentCollection; @@ -90,22 +92,22 @@ public async Task CreateTargetDocumentCollection(DocumentCli await targetClient.CreateDatabaseIfNotExistsAsync(new Database { Id = targetDatabaseName }); DocumentCollection newDocumentCollection; - if (partitionKeyDefinition != null && partitionKeyDefinition.Paths.Count>0) + if (partitionKeyDefinition != null && partitionKeyDefinition.Paths.Count > 0) { - if(CloneSettings.CopyPartitionKey) - { - // Partition key exists in Source (Unlimited Storage) - newDocumentCollection = (DocumentCollection)await targetClient.CreateDocumentCollectionIfNotExistsAsync - (UriFactory.CreateDatabaseUri(targetDatabaseName), - new DocumentCollection { Id = targetCollectionName, PartitionKey = partitionKeyDefinition, IndexingPolicy = indexingPolicy }, - new RequestOptions { OfferEnableRUPerMinuteThroughput = true, OfferThroughput = CloneSettings.TargetMigrationOfferThroughputRUs }); + if (CloneSettings.CopyPartitionKey) + { + // Partition key exists in Source (Unlimited Storage) + newDocumentCollection = (DocumentCollection)await targetClient.CreateDocumentCollectionIfNotExistsAsync + (UriFactory.CreateDatabaseUri(targetDatabaseName), + new DocumentCollection { Id = targetCollectionName, PartitionKey = partitionKeyDefinition, IndexingPolicy = indexingPolicy }, + new RequestOptions { OfferEnableRUPerMinuteThroughput = true, OfferThroughput = CloneSettings.TargetMigrationOfferThroughputRUs }); } else { - newDocumentCollection = (DocumentCollection)await targetClient.CreateDocumentCollectionIfNotExistsAsync - (UriFactory.CreateDatabaseUri(targetDatabaseName), - new DocumentCollection { Id = targetCollectionName, IndexingPolicy = indexingPolicy }, - new RequestOptions { OfferEnableRUPerMinuteThroughput = true, OfferThroughput = CloneSettings.TargetMigrationOfferThroughputRUs }); + newDocumentCollection = (DocumentCollection)await targetClient.CreateDocumentCollectionIfNotExistsAsync + (UriFactory.CreateDatabaseUri(targetDatabaseName), + new DocumentCollection { Id = targetCollectionName, IndexingPolicy = indexingPolicy }, + new RequestOptions { OfferEnableRUPerMinuteThroughput = true, OfferThroughput = CloneSettings.TargetMigrationOfferThroughputRUs }); } } else @@ -124,6 +126,6 @@ public async Task CreateTargetDocumentCollection(DocumentCli throw; } } - + } } diff --git a/CosmosClone/CloneConsoleRun/Sample/Entity.cs b/CosmosClone/CloneConsoleRun/Sample/Entity.cs index b3412c8..4f75ff5 100644 --- a/CosmosClone/CloneConsoleRun/Sample/Entity.cs +++ b/CosmosClone/CloneConsoleRun/Sample/Entity.cs @@ -6,6 +6,7 @@ namespace CloneConsoleRun.Sample { using System; using System.Collections.Generic; + using CosmosCloneCommon.Model; public class Entity @@ -99,9 +100,9 @@ public static Entitytest getRandomTestEntity() testEntity.Words.Add("rex123"); return testEntity; } - } + } + - diff --git a/CosmosClone/CloneConsoleRun/Sample/EntityV2.cs b/CosmosClone/CloneConsoleRun/Sample/EntityV2.cs index 0658f30..5f99b3d 100644 --- a/CosmosClone/CloneConsoleRun/Sample/EntityV2.cs +++ b/CosmosClone/CloneConsoleRun/Sample/EntityV2.cs @@ -5,10 +5,11 @@ namespace CloneConsoleRun.Sample { using System; using System.Collections.Generic; + using CosmosCloneCommon.Model; public class EntityV2 - { + { public string FullName { get; set; } public string Description { get; set; } public Boolean IsActive { get; set; } @@ -30,7 +31,7 @@ public static EntityV2 getRandomEntity() entity.Description = "Test Sample Description " + entity.SuperId.ToString(); entity.EntityType = RandomNumberGenerator.GetRandomEntityType(); var employeeid = RandomNumberGenerator.GetNext(); - entity.EmailAddress = "intialTest"+ employeeid .ToString()+ "@test.com"; + entity.EmailAddress = "intialTest" + employeeid.ToString() + "@test.com"; entity.PhoneNumber = RandomNumberGenerator.GetNext().ToString(); entity.IsActive = true; entity.ModifiedTime = DateTime.UtcNow; @@ -46,7 +47,7 @@ public static EntityV2 getRandomEntity() } - + diff --git a/CosmosClone/CloneConsoleRun/Sample/SampleDBCreator.cs b/CosmosClone/CloneConsoleRun/Sample/SampleDBCreator.cs index 49741dc..28131b5 100644 --- a/CosmosClone/CloneConsoleRun/Sample/SampleDBCreator.cs +++ b/CosmosClone/CloneConsoleRun/Sample/SampleDBCreator.cs @@ -12,7 +12,9 @@ namespace CloneConsoleRun.Sample using System.Diagnostics; using System.Linq; using System.Threading.Tasks; + using CosmosCloneCommon.Utility; + using Microsoft.Azure.CosmosDB.BulkExecutor.BulkImport; using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; @@ -33,7 +35,7 @@ public class SampleDBCreator public SampleDBCreator() { cosmosHelper = new CosmosSampleDBHelper(); - cosmosBulkImporter = new CosmosBulkImporter(); + cosmosBulkImporter = new CosmosBulkImporter(); } public async Task InitializeMigration() @@ -53,7 +55,7 @@ public async Task Start() protected List GetCommonEntitiesinBatch() { List entities = new List(); - for(int i=0; i - - - - - - - - - - - \ No newline at end of file diff --git a/CosmosClone/CosmicCloneUI/App.config b/CosmosClone/CosmicCloneUI/App.config index 0795a74..1e15f6d 100644 --- a/CosmosClone/CosmicCloneUI/App.config +++ b/CosmosClone/CosmicCloneUI/App.config @@ -1,81 +1,65 @@ - + -
-
-
+
+
+
- - - - - + + + + + - - - - - + + + + + - - - - - - - + + + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - + - + - \ No newline at end of file + diff --git a/CosmosClone/CosmicCloneUI/App.xaml.cs b/CosmosClone/CosmicCloneUI/App.xaml.cs index 36308d4..d50c6bd 100644 --- a/CosmosClone/CosmicCloneUI/App.xaml.cs +++ b/CosmosClone/CosmicCloneUI/App.xaml.cs @@ -2,12 +2,6 @@ // Licensed under the MIT License. -using System; -using System.Collections.Generic; -using System.Configuration; -using System.Data; -using System.Linq; -using System.Threading.Tasks; using System.Windows; namespace CosmicCloneUI diff --git a/CosmosClone/CosmicCloneUI/CloneOptionsPage.xaml.cs b/CosmosClone/CosmicCloneUI/CloneOptionsPage.xaml.cs index df68cdb..a743fb2 100644 --- a/CosmosClone/CosmicCloneUI/CloneOptionsPage.xaml.cs +++ b/CosmosClone/CosmicCloneUI/CloneOptionsPage.xaml.cs @@ -2,19 +2,8 @@ // Licensed under the MIT License. using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Windows; using System.Windows.Controls; -using System.Windows.Data; -using System.Windows.Documents; -using System.Windows.Input; -using System.Windows.Media; using System.Windows.Media.Imaging; -using System.Windows.Navigation; -using System.Windows.Shapes; namespace CosmicCloneUI { diff --git a/CosmosClone/CosmicCloneUI/CopyCollectionPage.xaml.cs b/CosmosClone/CosmicCloneUI/CopyCollectionPage.xaml.cs index b347e17..39c080a 100644 --- a/CosmosClone/CosmicCloneUI/CopyCollectionPage.xaml.cs +++ b/CosmosClone/CosmicCloneUI/CopyCollectionPage.xaml.cs @@ -1,23 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using System; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using System.Windows; using System.Windows.Controls; -using System.Windows.Data; -using System.Windows.Documents; -using System.Windows.Input; -using System.Windows.Media; -using System.Windows.Media.Imaging; -using System.Windows.Navigation; -using System.Windows.Shapes; -using CosmosCloneCommon.Utility; -using logger = CosmosCloneCommon.Utility.CloneLogger; + using CosmosCloneCommon.Model; +using CosmosCloneCommon.Utility; namespace CosmicCloneUI { @@ -35,19 +24,19 @@ public CopyCollectionPage() public void SetStatus() { //StatusTextBlock.Text = CloneLogger.GetFullLog(); - + } public void setRequiredprogressBars(List scrubRules) { - if(CloneSettings.CopyDocuments==false) + if (CloneSettings.CopyDocuments == false) { CollectionReadStackPanel.Visibility = Visibility.Hidden; CollectionWriteStackPanel.Visibility = Visibility.Hidden; } - if(CloneSettings.ScrubbingRequired) + if (CloneSettings.ScrubbingRequired) { - if(scrubRules == null || scrubRules.Count<=0) + if (scrubRules == null || scrubRules.Count <= 0) { ScrubStackPanel.Visibility = Visibility.Hidden; } @@ -56,19 +45,19 @@ public void setRequiredprogressBars(List scrubRules) private void ShowStatusButton_Click(object sender, RoutedEventArgs e) { - if(StatusTextBlock.Visibility == Visibility.Visible) + if (StatusTextBlock.Visibility == Visibility.Visible) { StatusTextBlock.Visibility = Visibility.Hidden; StatusTextBlockBorder.Visibility = Visibility.Hidden; ShowStatusButton.Content = "Show status"; } - else if(StatusTextBlock.Visibility == Visibility.Hidden) + else if (StatusTextBlock.Visibility == Visibility.Hidden) { StatusTextBlock.Visibility = Visibility.Visible; StatusTextBlockBorder.Visibility = Visibility.Visible; ShowStatusButton.Content = "Hide status"; } - + } } } diff --git a/CosmosClone/CosmicCloneUI/CosmicCloneUI.csproj b/CosmosClone/CosmicCloneUI/CosmicCloneUI.csproj index ff3355e..e8d08bd 100644 --- a/CosmosClone/CosmicCloneUI/CosmicCloneUI.csproj +++ b/CosmosClone/CosmicCloneUI/CosmicCloneUI.csproj @@ -8,7 +8,7 @@ WinExe CosmicCloneUI CosmicCloneUI - v4.6.1 + v4.8 512 {60dc8134-eba5-43b8-bcc9-bb4bc16c2548};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} 4 @@ -29,6 +29,7 @@ false true true + AnyCPU @@ -70,14 +71,8 @@ - - - - - - 4.0 diff --git a/CosmosClone/CosmicCloneUI/DataAnonymizationPage.xaml.cs b/CosmosClone/CosmicCloneUI/DataAnonymizationPage.xaml.cs index b57a9ec..9f1a4fb 100644 --- a/CosmosClone/CosmicCloneUI/DataAnonymizationPage.xaml.cs +++ b/CosmosClone/CosmicCloneUI/DataAnonymizationPage.xaml.cs @@ -1,27 +1,21 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using CosmicCloneUI.Models; -using CosmosCloneCommon.Model; -using CosmosCloneCommon.Utility; using System; -using System.IO; using System.Collections.Generic; +using System.Globalization; +using System.IO; using System.Linq; -using System.Text; -using System.Threading.Tasks; using System.Windows; using System.Windows.Controls; -using System.Windows.Data; using System.Windows.Documents; -using System.Windows.Input; using System.Windows.Media; using System.Windows.Media.Imaging; -using System.Windows.Navigation; -using System.Windows.Shapes; + +using CosmosCloneCommon.Model; +using CosmosCloneCommon.Utility; + using Microsoft.Win32; -using System.Xml.Serialization; -using System.Globalization; namespace CosmicCloneUI { @@ -52,7 +46,7 @@ private void BtnAddScrubRule(object sender, RoutedEventArgs e) private void CreateScrubRule(ScrubRule scrubRule = null) { - WrapPanel parentStackPanelLeft = (WrapPanel) this.FindName("WrapPanel"); + WrapPanel parentStackPanelLeft = (WrapPanel)this.FindName("WrapPanel"); int newRuleIndex = newIndex(scrubRule); AddRuleWithData(parentStackPanelLeft, newRuleIndex, scrubRule); @@ -61,7 +55,7 @@ private void CreateScrubRule(ScrubRule scrubRule = null) int newIndex(ScrubRule scrubRule = null) { int newRuleIndex = 0; - if(scrubRule == null) + if (scrubRule == null) { newRuleIndex = RuleIndex++; } @@ -239,20 +233,20 @@ void AddRuleWithData(WrapPanel parentStackPanel, int ruleIndex, ScrubRule scrubR exp.Content = sp; - if(scrubRule!=null) + if (scrubRule != null) { FilterTB.Text = scrubRule.FilterCondition; AttributeScrubTB.Text = scrubRule.PropertyName; - if(scrubRule.Type!=null) ScrubTypeCB.SelectedIndex = (int)scrubRule.Type; + if (scrubRule.Type != null) ScrubTypeCB.SelectedIndex = (int)scrubRule.Type; ScrubValueTB.Text = scrubRule.UpdateValue; } parentStackPanel.Children.Add(exp); - if(!SaveRuleButton.IsEnabled) + if (!SaveRuleButton.IsEnabled) { SaveRuleButton.IsEnabled = true; } } - + private void scrubTypeComboBox_SelectedIndexChanged(object sender, EventArgs e) { @@ -276,7 +270,7 @@ private void scrubTypeComboBox_SelectedIndexChanged(object sender, EventArgs e) private void DeleteBtn_Click(object sender, RoutedEventArgs e) { var btnDelete = (Button)sender; - string expname = "RuleExpander_" + btnDelete.Name.Substring(btnDelete.Name.IndexOf('_')+1); + string expname = "RuleExpander_" + btnDelete.Name.Substring(btnDelete.Name.IndexOf('_') + 1); WrapPanel wrapPanel = (WrapPanel)this.FindName("WrapPanel"); foreach (UIElement SPUI in wrapPanel.Children) @@ -300,35 +294,35 @@ private void DeleteBtn_Click(object sender, RoutedEventArgs e) { rule.RuleId = 0;//reset Ids so they are re assigned CreateScrubRule(rule); - } + } } public List getScrubRules() { //List sb = new List(); - TextBox filterCondition = (TextBox) this.FindName("FilterCondition"); + TextBox filterCondition = (TextBox)this.FindName("FilterCondition"); //sb.filterQuery = filterCondition.Text; List srList = new List(); - WrapPanel wrapPanel = (WrapPanel) this.FindName("WrapPanel"); + WrapPanel wrapPanel = (WrapPanel)this.FindName("WrapPanel"); foreach (UIElement SPUI in wrapPanel.Children) { - Expander exp = (Expander) SPUI; - StackPanel lrsp = (StackPanel) exp.Content; + Expander exp = (Expander)SPUI; + StackPanel lrsp = (StackPanel)exp.Content; UIElementCollection uiElementsSP = lrsp.Children; ScrubRule sr = new ScrubRule(); foreach (UIElement uiElementSP in uiElementsSP) { - StackPanel tempSP = (StackPanel) uiElementSP; + StackPanel tempSP = (StackPanel)uiElementSP; UIElementCollection uiElements = tempSP.Children; foreach (UIElement uiElement in uiElements) { if (uiElement.GetType().Name == "Label") { - var ruleIdLabel = (Label) uiElement; + var ruleIdLabel = (Label)uiElement; int ruleId; if (int.TryParse(ruleIdLabel.Content.ToString(), out ruleId)) { @@ -339,7 +333,7 @@ public List getScrubRules() if (uiElement.GetType().Name == "TextBox") { - TextBox tb = (TextBox) uiElement; + TextBox tb = (TextBox)uiElement; if (tb.Name.StartsWith("Filter")) { @@ -357,7 +351,7 @@ public List getScrubRules() if (uiElement.GetType().Name == "ComboBox") { - ComboBox cb = (ComboBox) uiElement; + ComboBox cb = (ComboBox)uiElement; if (cb.Name.StartsWith("ScrubType")) { //sr.Type = (RuleType) Enum.Parse(typeof(RuleType), cb.Text); @@ -392,11 +386,11 @@ public bool validateInput() foreach (var rule in orderedRules) { - if(rule.Type == null) + if (rule.Type == null) { validationMessages.Add($"Rule:{rule.RuleId} - Please select a valid anonymization type"); } - if(string.IsNullOrEmpty(rule.PropertyName)) + if (string.IsNullOrEmpty(rule.PropertyName)) { validationMessages.Add($"Rule:{rule.RuleId} - Attribute name is empty"); } @@ -414,17 +408,17 @@ public bool validateInput() { validationMessages.Add($"Rule:{rule.RuleId} - Filter condition starts improperly. Sample c.EntityType=\"document\" "); } - } + } } - if(validationMessages.Count > 0) + if (validationMessages.Count > 0) { isValidationSuccess = false; - string message=""; - foreach(var msg in validationMessages) + string message = ""; + foreach (var msg in validationMessages) { message += (msg + System.Environment.NewLine); - } - MessageBox.Show(message, "Input validation",MessageBoxButton.OK,MessageBoxImage.Error); + } + MessageBox.Show(message, "Input validation", MessageBoxButton.OK, MessageBoxImage.Error); } else isValidationSuccess = true; return isValidationSuccess; @@ -438,7 +432,7 @@ private void SaveRuleButton_Click(object sender, RoutedEventArgs e) if (rules == null || rules.Count == 0) { - MessageBox.Show("No Rules found. Please add/load anonymization rules before Save", "No rules Found", MessageBoxButton.OK, MessageBoxImage.Warning); + MessageBox.Show("No Rules found. Please add/load anonymization rules before Save", "No rules Found", MessageBoxButton.OK, MessageBoxImage.Warning); return; } @@ -446,13 +440,13 @@ private void SaveRuleButton_Click(object sender, RoutedEventArgs e) saveFileDialog.InitialDirectory = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments); saveFileDialog.Filter = "XML file (*.xml)|*.xml"; saveFileDialog.Title = "CosmicClone save AnonymizationRules"; - saveFileDialog.FileName = "AnonymizationRules_"+ DateTime.Now.ToString("MM-dd-yyyy-HHmmss",CultureInfo.InvariantCulture); - + saveFileDialog.FileName = "AnonymizationRules_" + DateTime.Now.ToString("MM-dd-yyyy-HHmmss", CultureInfo.InvariantCulture); + if (saveFileDialog.ShowDialog() == true) - { + { var xmlText = CloneSerializer.XMLSerialize(rules); File.WriteAllText(saveFileDialog.FileName, xmlText); - } + } } private void LoadRuleButton_Click(object sender, RoutedEventArgs e) @@ -467,12 +461,12 @@ private void LoadRuleButton_Click(object sender, RoutedEventArgs e) var rules = CloneSerializer.XMLDeserialize>(xmlText); if (rules == null && rules.Count == 0) { - MessageBox.Show("No rules to Load in file : "+openFileDialog.FileName , "No rules Found", MessageBoxButton.OK, MessageBoxImage.Warning); + MessageBox.Show("No rules to Load in file : " + openFileDialog.FileName, "No rules Found", MessageBoxButton.OK, MessageBoxImage.Warning); return; } var orderedRules = rules.OrderBy(o => o.RuleId).ToList(); - + //Delete all scrub rules WrapPanel wrapPanel = (WrapPanel)this.FindName("WrapPanel"); wrapPanel.Children.Clear(); @@ -492,7 +486,7 @@ private void LoadRuleButton_Click(object sender, RoutedEventArgs e) private void ValidateRuleButton_Click(object sender, RoutedEventArgs e) { var rules = getScrubRules(); - if (rules == null || rules.Count==0) + if (rules == null || rules.Count == 0) { MessageBox.Show("No Rules found. Please add/load anonymization rules before Validation", "Data validation", MessageBoxButton.OK, MessageBoxImage.Information); return; diff --git a/CosmosClone/CosmicCloneUI/DestinationPage.xaml.cs b/CosmosClone/CosmicCloneUI/DestinationPage.xaml.cs index 84da4a1..e2f8dff 100644 --- a/CosmosClone/CosmicCloneUI/DestinationPage.xaml.cs +++ b/CosmosClone/CosmicCloneUI/DestinationPage.xaml.cs @@ -1,21 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using CosmosCloneCommon.Utility; using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using System.Windows; using System.Windows.Controls; -using System.Windows.Data; -using System.Windows.Documents; -using System.Windows.Input; -using System.Windows.Media; using System.Windows.Media.Imaging; -using System.Windows.Navigation; -using System.Windows.Shapes; + +using CosmosCloneCommon.Utility; namespace CosmicCloneUI { diff --git a/CosmosClone/CosmicCloneUI/MainWindow.xaml b/CosmosClone/CosmicCloneUI/MainWindow.xaml index 55e6013..db6bd5c 100644 --- a/CosmosClone/CosmicCloneUI/MainWindow.xaml +++ b/CosmosClone/CosmicCloneUI/MainWindow.xaml @@ -5,7 +5,7 @@ xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" xmlns:local="clr-namespace:CosmicCloneUI" mc:Ignorable="d" - Title="CosmicClone v2.0" Height="500" Width="797" ResizeMode="NoResize" WindowStartupLocation="CenterScreen" Icon="/Images/CosmosIcon2.ico"> + Title="CosmicClone v2.5" Height="500" Width="797" ResizeMode="NoResize" WindowStartupLocation="CenterScreen" Icon="/Images/CosmosIcon2.ico"> diff --git a/CosmosClone/CosmicCloneUI/MainWindow.xaml.cs b/CosmosClone/CosmicCloneUI/MainWindow.xaml.cs index 0434c17..990eb8c 100644 --- a/CosmosClone/CosmicCloneUI/MainWindow.xaml.cs +++ b/CosmosClone/CosmicCloneUI/MainWindow.xaml.cs @@ -1,25 +1,17 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using CosmicCloneUI.Models; -using CosmosCloneCommon.Utility; using System; using System.Collections.Generic; -using System.Linq; -using System.Text; +using System.ComponentModel; using System.Threading.Tasks; using System.Windows; using System.Windows.Controls; -using System.Windows.Data; -using System.Windows.Documents; -using System.Windows.Input; -using System.Windows.Media; -using System.Windows.Media.Imaging; -using System.Windows.Navigation; -using System.Windows.Shapes; -using System.ComponentModel; + using CosmosCloneCommon.Migrator; using CosmosCloneCommon.Model; +using CosmosCloneCommon.Utility; + using logger = CosmosCloneCommon.Utility.CloneLogger; namespace CosmicCloneUI @@ -69,7 +61,7 @@ private void InitializePages() pages[3] = dataAnonymizationPage; pages[4] = copyCollectionPage; - _mainFrame.Content = pages[0]; + _mainFrame.Content = pages[0]; } private void BtnClickPrevious(object sender, RoutedEventArgs e) @@ -81,10 +73,10 @@ private void BtnClickPrevious(object sender, RoutedEventArgs e) private void BtnClickNext(object sender, RoutedEventArgs e) { Page currentPage = (Page)_mainFrame.Content; - if(PerformAction(currentPage)) + if (PerformAction(currentPage)) { _mainFrame.Navigate(GetNextPage(currentPage)); - } + } } private void BtnClickFinish(object sender, RoutedEventArgs e) @@ -128,9 +120,9 @@ private Page GetPreviousPage(Page currentPage) private int GetPageNumber(Page page) { - for(int i=0;i 0) { scrubPercentProgress = DocumentMigrator.ScrubPercentProgress; - - } + + } else { if (DocumentMigrator.scrubRules == null || DocumentMigrator.scrubRules.Count == 0) scrubPercentProgress = 100; @@ -287,7 +288,7 @@ void Worker_DoWork2(object sender, DoWorkEventArgs e) } - sendPercent = (int)scrubPercentProgress * 1000000 + (int)readPercentProgress * 1000 + (int)writePercentProgress ; + sendPercent = (int)scrubPercentProgress * 1000000 + (int)readPercentProgress * 1000 + (int)writePercentProgress; (sender as BackgroundWorker).ReportProgress((int)sendPercent); Task.Delay(3000).Wait(); } @@ -312,24 +313,35 @@ void Worker_ProgressChanged2(object sender, ProgressChangedEventArgs e) void Worker_RunWorkerCompleted2(object sender, RunWorkerCompletedEventArgs e) { - while(!DocumentMigrator.IsCodeMigrationComplete) + while (!DocumentMigrator.IsCodeMigrationComplete) { Task.Delay(5000).Wait(); - } - if(DocumentMigrator.IsCodeMigrationComplete) + } + if (DocumentMigrator.IsCodeMigrationComplete) { string completeMessage = DocumentMigrator.TotalRecordsSent + " Documents Copied Successfully"; completeMessage += "\n" + "Code Migration Complete"; - if (DocumentMigrator.scrubRules!=null && DocumentMigrator.scrubRules.Count>0) + if (DocumentMigrator.scrubRules != null && DocumentMigrator.scrubRules.Count > 0) { completeMessage += "\n" + "Scrubbing completed for rules " + DocumentMigrator.scrubRules.Count; } - + MessageBox.Show(completeMessage, "Completed", MessageBoxButton.OK, MessageBoxImage.Information); - btn_finish.IsEnabled = true; + string copyMoreMessage = "Would you like to copy additional collections?"; + var wantToCopyMore = MessageBox.Show(copyMoreMessage, "Copy more?", MessageBoxButton.YesNo, MessageBoxImage.Question); + switch (wantToCopyMore) + { + case MessageBoxResult.Yes: + DoAnotherCopy(); + break; + case MessageBoxResult.Cancel: + case MessageBoxResult.No: + default: + btn_finish.IsEnabled = true; + break; + } } } - } } diff --git a/CosmosClone/CosmicCloneUI/Models/CloneOptions.cs b/CosmosClone/CosmicCloneUI/Models/CloneOptions.cs index ae44dfb..21d3455 100644 --- a/CosmosClone/CosmicCloneUI/Models/CloneOptions.cs +++ b/CosmosClone/CosmicCloneUI/Models/CloneOptions.cs @@ -1,13 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - namespace CosmicCloneUI.Models { class CloneOptions diff --git a/CosmosClone/CosmicCloneUI/Models/CosmosCollection.cs b/CosmosClone/CosmicCloneUI/Models/CosmosCollection.cs index cd632fa..3376fb7 100644 --- a/CosmosClone/CosmicCloneUI/Models/CosmosCollection.cs +++ b/CosmosClone/CosmicCloneUI/Models/CosmosCollection.cs @@ -1,12 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - namespace CosmicCloneUI.Models { class CosmosCollection diff --git a/CosmosClone/CosmicCloneUI/Models/ScrubDetails.cs b/CosmosClone/CosmicCloneUI/Models/ScrubDetails.cs index 23717b9..aee653a 100644 --- a/CosmosClone/CosmicCloneUI/Models/ScrubDetails.cs +++ b/CosmosClone/CosmicCloneUI/Models/ScrubDetails.cs @@ -1,13 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - namespace CosmicCloneUI.Models { //class ScrubDetails diff --git a/CosmosClone/CosmicCloneUI/Properties/AssemblyInfo.cs b/CosmosClone/CosmicCloneUI/Properties/AssemblyInfo.cs index 212e420..496b223 100644 --- a/CosmosClone/CosmicCloneUI/Properties/AssemblyInfo.cs +++ b/CosmosClone/CosmicCloneUI/Properties/AssemblyInfo.cs @@ -1,6 +1,4 @@ using System.Reflection; -using System.Resources; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Windows; diff --git a/CosmosClone/CosmicCloneUI/Properties/Resources.Designer.cs b/CosmosClone/CosmicCloneUI/Properties/Resources.Designer.cs index a480252..c920aee 100644 --- a/CosmosClone/CosmicCloneUI/Properties/Resources.Designer.cs +++ b/CosmosClone/CosmicCloneUI/Properties/Resources.Designer.cs @@ -8,10 +8,10 @@ // //------------------------------------------------------------------------------ -namespace CosmicCloneUI.Properties -{ - - +namespace CosmicCloneUI.Properties { + using System; + + /// /// A strongly-typed resource class, for looking up localized strings, etc. /// @@ -19,51 +19,43 @@ namespace CosmicCloneUI.Properties // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] - internal class Resources - { - + internal class Resources { + private static global::System.Resources.ResourceManager resourceMan; - + private static global::System.Globalization.CultureInfo resourceCulture; - + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] - internal Resources() - { + internal Resources() { } - + /// /// Returns the cached ResourceManager instance used by this class. /// [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] - internal static global::System.Resources.ResourceManager ResourceManager - { - get - { - if ((resourceMan == null)) - { + internal static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("CosmicCloneUI.Properties.Resources", typeof(Resources).Assembly); resourceMan = temp; } return resourceMan; } } - + /// /// Overrides the current thread's CurrentUICulture property for all /// resource lookups using this strongly typed resource class. /// [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] - internal static global::System.Globalization.CultureInfo Culture - { - get - { + internal static global::System.Globalization.CultureInfo Culture { + get { return resourceCulture; } - set - { + set { resourceCulture = value; } } diff --git a/CosmosClone/CosmicCloneUI/Properties/Settings.Designer.cs b/CosmosClone/CosmicCloneUI/Properties/Settings.Designer.cs index 93fd918..07e772e 100644 --- a/CosmosClone/CosmicCloneUI/Properties/Settings.Designer.cs +++ b/CosmosClone/CosmicCloneUI/Properties/Settings.Designer.cs @@ -8,21 +8,17 @@ // //------------------------------------------------------------------------------ -namespace CosmicCloneUI.Properties -{ - - +namespace CosmicCloneUI.Properties { + + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "11.0.0.0")] - internal sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase - { - + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("Microsoft.VisualStudio.Editors.SettingsDesigner.SettingsSingleFileGenerator", "17.10.0.0")] + internal sealed partial class Settings : global::System.Configuration.ApplicationSettingsBase { + private static Settings defaultInstance = ((Settings)(global::System.Configuration.ApplicationSettingsBase.Synchronized(new Settings()))); - - public static Settings Default - { - get - { + + public static Settings Default { + get { return defaultInstance; } } diff --git a/CosmosClone/CosmicCloneUI/SourcePage.xaml.cs b/CosmosClone/CosmicCloneUI/SourcePage.xaml.cs index 8ce0b49..100459e 100644 --- a/CosmosClone/CosmicCloneUI/SourcePage.xaml.cs +++ b/CosmosClone/CosmicCloneUI/SourcePage.xaml.cs @@ -1,18 +1,9 @@ -using CosmosCloneCommon.Utility; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using System; using System.Windows; using System.Windows.Controls; -using System.Windows.Data; -using System.Windows.Documents; -using System.Windows.Input; -using System.Windows.Media; using System.Windows.Media.Imaging; -using System.Windows.Navigation; -using System.Windows.Shapes; + +using CosmosCloneCommon.Utility; namespace CosmicCloneUI { diff --git a/CosmosClone/CosmosCloneCommon/CosmosCloneCommon.csproj b/CosmosClone/CosmosCloneCommon/CosmosCloneCommon.csproj index cea6943..927d210 100644 --- a/CosmosClone/CosmosCloneCommon/CosmosCloneCommon.csproj +++ b/CosmosClone/CosmosCloneCommon/CosmosCloneCommon.csproj @@ -9,10 +9,11 @@ Properties CosmosCloneCommon CosmosCloneCommon - v4.6.1 + v4.8 512 + true @@ -32,43 +33,14 @@ 4 - - ..\packages\Microsoft.Azure.CosmosDB.BulkExecutor.1.2.0\lib\net451\Microsoft.Azure.CosmosDB.BulkImport.dll - - - ..\packages\Microsoft.Azure.DocumentDB.2.1.3\lib\net461\Microsoft.Azure.Documents.Client.dll - - - ..\packages\MongoDB.Bson.2.4.4\lib\net45\MongoDB.Bson.dll - - - ..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll - - - ..\packages\System.Buffers.4.5.0\lib\netstandard2.0\System.Buffers.dll - - - ..\packages\System.Memory.4.5.0\lib\netstandard2.0\System.Memory.dll - - - ..\packages\System.Net.Http.WinHttpHandler.4.5.0\lib\net461\System.Net.Http.WinHttpHandler.dll - - - ..\packages\System.Numerics.Vectors.4.4.0\lib\net46\System.Numerics.Vectors.dll - - - ..\packages\System.Runtime.CompilerServices.Unsafe.4.5.0\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll - - - @@ -90,17 +62,23 @@ Designer - - Designer - - + + + 1.8.9 + + + 2.24.0 + + + 2.19.0 + + + 13.0.3 + + + 6.0.0 + + - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - \ No newline at end of file diff --git a/CosmosClone/CosmosCloneCommon/Migrator/CodeMigrator.cs b/CosmosClone/CosmosCloneCommon/Migrator/CodeMigrator.cs index 98cc2cf..7512770 100644 --- a/CosmosClone/CosmosCloneCommon/Migrator/CodeMigrator.cs +++ b/CosmosClone/CosmosCloneCommon/Migrator/CodeMigrator.cs @@ -5,13 +5,15 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading.Tasks; -using logger = CosmosCloneCommon.Utility.CloneLogger; + using CosmosCloneCommon.Utility; + using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; +using logger = CosmosCloneCommon.Utility.CloneLogger; + namespace CosmosCloneCommon.Migrator { public class CodeMigrator @@ -40,7 +42,7 @@ public CodeMigrator() { //initialize settings and other utilities var SourceCosmosDBSettings = CloneSettings.GetConfigurationSection("SourceCosmosDBSettings"); - SourceEndpointUrl = CloneSettings.SourceSettings.EndpointUrl; ; + SourceEndpointUrl = CloneSettings.SourceSettings.EndpointUrl; SourceAccessKey = CloneSettings.SourceSettings.AccessKey; sourceDatabaseName = CloneSettings.SourceSettings.DatabaseName; sourceCollectionName = CloneSettings.SourceSettings.CollectionName; diff --git a/CosmosClone/CosmosCloneCommon/Migrator/DataScrubMigrator.cs b/CosmosClone/CosmosCloneCommon/Migrator/DataScrubMigrator.cs index 5e1a094..e849e9d 100644 --- a/CosmosClone/CosmosCloneCommon/Migrator/DataScrubMigrator.cs +++ b/CosmosClone/CosmosCloneCommon/Migrator/DataScrubMigrator.cs @@ -4,20 +4,23 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; -using System.Text; -using Newtonsoft.Json; using System.Threading.Tasks; + +using CosmosCloneCommon.Model; +using CosmosCloneCommon.Utility; + +using Microsoft.Azure.CosmosDB.BulkExecutor.BulkImport; using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; -using System.Diagnostics; -using Microsoft.Azure.CosmosDB.BulkExecutor.BulkImport; using Microsoft.Azure.Documents.Linq; -using CosmosCloneCommon.Utility; -using CosmosCloneCommon.Model; -using logger = CosmosCloneCommon.Utility.CloneLogger; + +using Newtonsoft.Json; using Newtonsoft.Json.Linq; +using logger = CosmosCloneCommon.Utility.CloneLogger; + namespace CosmosCloneCommon.Migrator { public class DataScrubMigrator @@ -67,9 +70,9 @@ public async Task StartScrub(List scrubRules) ScrubDataFetchQuery = cosmosHelper.GetScrubDataDocumentQuery(targetClient, filterCondition, CloneSettings.ReadBatchSize); await ReadUploadInBatches((IDocumentQuery)ScrubDataFetchQuery, sRules); - foreach(var srule in DataScrubMigrator.scrubRules) + foreach (var srule in DataScrubMigrator.scrubRules) { - if(srule.FilterCondition.Equals(filterCondition)) + if (srule.FilterCondition.Equals(filterCondition)) { srule.IsComplete = true; srule.RecordsByFilter = filterRecordCount; @@ -86,7 +89,7 @@ public async Task InitializeMigration() targetClient = cosmosHelper.GetTargetDocumentDbClient(); targetCollection = await cosmosHelper.GetTargetDocumentCollection(targetClient); await cosmosBulkImporter.InitializeBulkExecutor(targetClient, targetCollection); - } + } public async Task ReadUploadInBatches(IDocumentQuery query, List scrubRules) { @@ -121,17 +124,17 @@ public async Task ReadUploadInBatches(IDocumentQuery query, List().ToList(); try { uploadResponse = await cosmosBulkImporter.BulkSendToNewCollection(objEntities); } - catch(Exception ex) + catch (Exception ex) { logger.LogError(ex); - throw; - } + throw; + } } badEntities = uploadResponse.BadInputDocuments; TotalRecordsScrubbed += uploadResponse.NumberOfDocumentsImported; @@ -160,9 +163,9 @@ protected async Task> GetCommonStringEntitiesinBatch(IDocumentQuery objEntities.AddRange((IEnumerable)res); logger.LogInfo($"Records retrieved from source: {objEntities.Count - prevRecordCount}"); } - foreach(var obj in objEntities) - { - entities.Add(JsonConvert.SerializeObject(obj)); + foreach (var obj in objEntities) + { + entities.Add(JsonConvert.SerializeObject(obj)); } logger.LogInfo($"Total Records retrieved from Source {entities.Count}"); return entities; diff --git a/CosmosClone/CosmosCloneCommon/Migrator/DocumentMigrator.cs b/CosmosClone/CosmosCloneCommon/Migrator/DocumentMigrator.cs index 7fdc9f5..b3df820 100644 --- a/CosmosClone/CosmosCloneCommon/Migrator/DocumentMigrator.cs +++ b/CosmosClone/CosmosCloneCommon/Migrator/DocumentMigrator.cs @@ -4,32 +4,34 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; -using System.Text; using System.Threading.Tasks; + +using CosmosCloneCommon.Model; +using CosmosCloneCommon.Utility; + +using Microsoft.Azure.CosmosDB.BulkExecutor.BulkImport; using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; -using System.Diagnostics; -using Microsoft.Azure.CosmosDB.BulkExecutor; -using Microsoft.Azure.CosmosDB.BulkExecutor.BulkImport; using Microsoft.Azure.Documents.Linq; -using CosmosCloneCommon.Utility; -using CosmosCloneCommon.Model; + using Newtonsoft.Json; -using logger = CosmosCloneCommon.Utility.CloneLogger; using Newtonsoft.Json.Linq; +using logger = CosmosCloneCommon.Utility.CloneLogger; + namespace CosmosCloneCommon.Migrator { public class DocumentMigrator { #region declare variables protected int ReadDelaybetweenRequestsInMs = 2000; - + protected CosmosDBHelper cosmosHelper; protected CosmosBulkImporter cosmosBulkImporter; protected DocumentClient sourceClient; - + protected DocumentClient targetClient; protected DocumentCollection sourceCollection; @@ -69,7 +71,7 @@ public static int ScrubPercentProgress } else if (IsInitialized) return 100; else return 0; - } + } } #endregion @@ -100,13 +102,13 @@ public async Task StartCopy(List scrubRules = null) var result = await dcs.StartScrub(noFilterScrubRules); } } - + if (CloneSettings.ScrubbingRequired && filteredScrubRules != null && filteredScrubRules.Count > 0) { var dcs = new DataScrubMigrator(); var result = await dcs.StartScrub(filteredScrubRules); } - + logger.LogScrubRulesInformation(DocumentMigrator.scrubRules); if (CloneSettings.CopyStoredProcedures) { await CopyStoredProcedures(); } @@ -127,8 +129,8 @@ public async Task InitializeMigration() sourceCollection = await cosmosHelper.GetSourceDocumentCollection(sourceClient); targetClient = cosmosHelper.GetTargetDocumentDbClient(); - var indexPolicy = (CloneSettings.CopyIndexingPolicy)? sourceCollection.IndexingPolicy : new IndexingPolicy(); - targetCollection = await cosmosHelper.CreateTargetDocumentCollection(targetClient, indexPolicy, sourceCollection.PartitionKey); + var indexPolicy = (CloneSettings.CopyIndexingPolicy) ? sourceCollection.IndexingPolicy : new IndexingPolicy(); + targetCollection = await cosmosHelper.CreateTargetDocumentCollection(targetClient, indexPolicy, sourceCollection.PartitionKey); if (CloneSettings.CopyDocuments) { @@ -148,7 +150,7 @@ public async Task InitializeMigration() filteredScrubRules = new List(); foreach (var sRule in scrubRules) { - if(string.IsNullOrEmpty(sRule.FilterCondition)) + if (string.IsNullOrEmpty(sRule.FilterCondition)) { sRule.RecordsByFilter = TotalRecordsInSource; noFilterScrubRules.Add(sRule); @@ -161,8 +163,8 @@ public async Task InitializeMigration() } } } - - public async Task ReadUploadInBatches(IDocumentQuery query) + + public async Task ReadUploadInBatches(IDocumentQuery query) { #region batchVariables @@ -192,13 +194,13 @@ public async Task ReadUploadInBatches(IDocumentQuery query) var scrubbedEntities = strEntities; if (entities.Any()) { - if( noFilterScrubRules == null || noFilterScrubRules.Count==0) - { + if (noFilterScrubRules == null || noFilterScrubRules.Count == 0) + { uploadResponse = await cosmosBulkImporter.BulkSendToNewCollection(entities); } else { - var jEntities = new List(); + var jEntities = new List(); foreach (var sRule in noFilterScrubRules) { jEntities = objectScrubber.ScrubObjectList(scrubbedEntities, sRule); @@ -212,7 +214,7 @@ public async Task ReadUploadInBatches(IDocumentQuery query) } var objDocuments = jEntities.Cast().ToList(); uploadResponse = await cosmosBulkImporter.BulkSendToNewCollection(objDocuments); - } + } } else { @@ -270,7 +272,7 @@ protected async Task> GetCommonEntitiesinBatch(IDocumentQuery Shuffle(List list) { - int n = list.Count-1; + int n = list.Count - 1; while (n > 1) { - n--; + n--; int k = RandomNumberGenerator.GetNext(n + 1); T value = list[k]; list[k] = list[n]; diff --git a/CosmosClone/CosmosCloneCommon/Model/ScrubRule.cs b/CosmosClone/CosmosCloneCommon/Model/ScrubRule.cs index 7cd528e..d4d7023 100644 --- a/CosmosClone/CosmosCloneCommon/Model/ScrubRule.cs +++ b/CosmosClone/CosmosCloneCommon/Model/ScrubRule.cs @@ -1,13 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - namespace CosmosCloneCommon.Model { public class ScrubRule @@ -15,7 +8,7 @@ public class ScrubRule public int RuleId { get; set; } public string FilterCondition { get; set; } public string PropertyName { get; set; } - public string UpdateValue { get; set; } + public string UpdateValue { get; set; } public RuleType? Type { get; set; } @@ -26,7 +19,7 @@ public class ScrubRule public ScrubRule() { } public ScrubRule(string filterCondition, string propertyName, RuleType type, string updateValue, int ruleId) - { + { this.FilterCondition = filterCondition; this.PropertyName = propertyName; @@ -35,7 +28,7 @@ public ScrubRule(string filterCondition, string propertyName, RuleType type, str this.RuleId = ruleId; this.IsComplete = false; } - + } public enum RuleType { SingleValue, NullValue, Shuffle, PartialMaskFromLeft, PartialMaskFromRight };//Can add random rule type later if required. diff --git a/CosmosClone/CosmosCloneCommon/Model/Validationresult.cs b/CosmosClone/CosmosCloneCommon/Model/Validationresult.cs index 224872d..303cd6e 100644 --- a/CosmosClone/CosmosCloneCommon/Model/Validationresult.cs +++ b/CosmosClone/CosmosCloneCommon/Model/Validationresult.cs @@ -1,13 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - namespace CosmosCloneCommon.Model { public class ValidationResult diff --git a/CosmosClone/CosmosCloneCommon/Properties/AssemblyInfo.cs b/CosmosClone/CosmosCloneCommon/Properties/AssemblyInfo.cs index cd66667..3ecda59 100644 --- a/CosmosClone/CosmosCloneCommon/Properties/AssemblyInfo.cs +++ b/CosmosClone/CosmosCloneCommon/Properties/AssemblyInfo.cs @@ -1,5 +1,4 @@ using System.Reflection; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; // General Information about an assembly is controlled through the following diff --git a/CosmosClone/CosmosCloneCommon/Utility/CloneLogger.cs b/CosmosClone/CosmosCloneCommon/Utility/CloneLogger.cs index 6049de1..6af8abf 100644 --- a/CosmosClone/CosmosCloneCommon/Utility/CloneLogger.cs +++ b/CosmosClone/CosmosCloneCommon/Utility/CloneLogger.cs @@ -4,10 +4,8 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Text; -using System.Threading.Tasks; -using System.Diagnostics; + using CosmosCloneCommon.Model; namespace CosmosCloneCommon.Utility @@ -24,7 +22,7 @@ public static string FullLog { get { - return _logBuilder.ToString(); + return _logBuilder.ToString(); } } @@ -36,7 +34,7 @@ public static string GetFullLog() public static void LogInfo(string info) { Console.WriteLine(info); - _logBuilder.Append("\n"+info); + _logBuilder.Append("\n" + info); } public static void LogError(string s) @@ -54,13 +52,13 @@ public static void LogError(Exception e) public static void LogScrubRulesInformation(List scrubRules) { - if (scrubRules!=null && scrubRules.Count>0 && CloneSettings.ScrubbingRequired) - { - long totalRecords=0; + if (scrubRules != null && scrubRules.Count > 0 && CloneSettings.ScrubbingRequired) + { + long totalRecords = 0; foreach (var rule in scrubRules) { LogInfo($"Rule Id: {rule.RuleId}. Attribute: {rule.PropertyName}"); - LogInfo($"Rule filter:{(string.IsNullOrEmpty(rule.FilterCondition) ? "None":rule.FilterCondition)}"); + LogInfo($"Rule filter:{(string.IsNullOrEmpty(rule.FilterCondition) ? "None" : rule.FilterCondition)}"); LogInfo($"Rule Type: {rule.Type.ToString()}"); LogInfo($"Records by filter: {rule.RecordsByFilter}. Records updated: {rule.RecordsUpdated}"); totalRecords += rule.RecordsUpdated; diff --git a/CosmosClone/CosmosCloneCommon/Utility/CloneSerializer.cs b/CosmosClone/CosmosCloneCommon/Utility/CloneSerializer.cs index fc52450..8263a66 100644 --- a/CosmosClone/CosmosCloneCommon/Utility/CloneSerializer.cs +++ b/CosmosClone/CosmosCloneCommon/Utility/CloneSerializer.cs @@ -2,11 +2,6 @@ // Licensed under the MIT License. -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using System.Xml.Serialization; namespace CosmosCloneCommon.Utility diff --git a/CosmosClone/CosmosCloneCommon/Utility/CloneSettings.cs b/CosmosClone/CosmosCloneCommon/Utility/CloneSettings.cs index 0eae3c1..609c9e2 100644 --- a/CosmosClone/CosmosCloneCommon/Utility/CloneSettings.cs +++ b/CosmosClone/CosmosCloneCommon/Utility/CloneSettings.cs @@ -2,17 +2,14 @@ // Licensed under the MIT License. using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using System.Configuration; using System.Collections.Specialized; +using System.Configuration; + using logger = CosmosCloneCommon.Utility.CloneLogger; namespace CosmosCloneCommon.Utility { - + public static class CloneSettings { public static bool CopyStoredProcedures { get; set; } @@ -25,7 +22,7 @@ public static class CloneSettings public static int WriteBatchSize { get; private set; } public static bool EnableTextLogging { get; set; } public static bool ScrubbingRequired { get; set; } - + public static int SourceOfferThroughputRUs { get; set; } public static int TargetMigrationOfferThroughputRUs { get; set; } public static int TargetRestOfferThroughputRUs { get; set; } @@ -53,7 +50,7 @@ static CloneSettings() TargetMigrationOfferThroughputRUs = int.Parse(ConfigurationManager.AppSettings["TargetMigrationOfferThroughputRUs"]); TargetRestOfferThroughputRUs = int.Parse(ConfigurationManager.AppSettings["TargetRestOfferThroughputRUs"]); ScrubbingRequired = bool.Parse(ConfigurationManager.AppSettings["ScrubbingRequired"]); - var sourceConfigs = GetConfigurationSection("SourceCosmosDBSettings"); + var sourceConfigs = GetConfigurationSection("SourceCosmosDBSettings"); SourceSettings = new CosmosCollectionValues() { EndpointUrl = sourceConfigs["EndpointUrl"], @@ -77,10 +74,12 @@ static CloneSettings() public static NameValueCollection GetConfigurationSection(string sectionName) { var appSettings = ConfigurationManager.GetSection(sectionName) as NameValueCollection; - if (appSettings.Count == 0) + + if (appSettings == null || appSettings.Count == 0) { - logger.LogInfo($"Application Settings are not defined for {sectionName}"); + logger.LogInfo($"Application Settings are {(appSettings == null ? "not defined" : "empty")} for {sectionName}"); } + return appSettings; } public static string AppSettings(string key) diff --git a/CosmosClone/CosmosCloneCommon/Utility/CosmosBulkImporter.cs b/CosmosClone/CosmosCloneCommon/Utility/CosmosBulkImporter.cs index 2305139..cb5a752 100644 --- a/CosmosClone/CosmosCloneCommon/Utility/CosmosBulkImporter.cs +++ b/CosmosClone/CosmosCloneCommon/Utility/CosmosBulkImporter.cs @@ -4,13 +4,14 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using System.Threading; +using System.Threading.Tasks; + using Microsoft.Azure.CosmosDB.BulkExecutor; using Microsoft.Azure.CosmosDB.BulkExecutor.BulkImport; using Microsoft.Azure.Documents; using Microsoft.Azure.Documents.Client; + using logger = CosmosCloneCommon.Utility.CloneLogger; namespace CosmosCloneCommon.Utility diff --git a/CosmosClone/CosmosCloneCommon/Utility/CosmosDBHelper.cs b/CosmosClone/CosmosCloneCommon/Utility/CosmosDBHelper.cs index be06081..5e6875e 100644 --- a/CosmosClone/CosmosCloneCommon/Utility/CosmosDBHelper.cs +++ b/CosmosClone/CosmosCloneCommon/Utility/CosmosDBHelper.cs @@ -2,13 +2,14 @@ // Licensed under the MIT License. using System; -using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading.Tasks; -using Microsoft.Azure.Documents.Client; -using Microsoft.Azure.Documents; + using CosmosCloneCommon.Model; + +using Microsoft.Azure.Documents; +using Microsoft.Azure.Documents.Client; + using logger = CosmosCloneCommon.Utility.CloneLogger; namespace CosmosCloneCommon.Utility @@ -100,10 +101,10 @@ public ValidationResult TestTargetConnection() public DocumentClient GetSourceDocumentDbClient() { try - { + { string SourceEndpointUrl = CloneSettings.SourceSettings.EndpointUrl; string SourceAccessKey = CloneSettings.SourceSettings.AccessKey; - var sourceDocumentClient = new DocumentClient(new Uri(SourceEndpointUrl), SourceAccessKey, ConnectionPolicy); + var sourceDocumentClient = new DocumentClient(new Uri(SourceEndpointUrl), SourceAccessKey, ConnectionPolicy); return sourceDocumentClient; } catch (Exception ex) @@ -238,22 +239,22 @@ public async Task CreateTargetDocumentCollection(DocumentCli await targetClient.CreateDatabaseIfNotExistsAsync(new Database { Id = targetDatabaseName }); DocumentCollection newDocumentCollection; - if (partitionKeyDefinition != null && partitionKeyDefinition.Paths.Count>0) + if (partitionKeyDefinition != null && partitionKeyDefinition.Paths.Count > 0) { - if(CloneSettings.CopyPartitionKey) - { - // Partition key exists in Source (Unlimited Storage) - newDocumentCollection = (DocumentCollection)await targetClient.CreateDocumentCollectionIfNotExistsAsync - (UriFactory.CreateDatabaseUri(targetDatabaseName), - new DocumentCollection { Id = targetCollectionName, PartitionKey = partitionKeyDefinition, IndexingPolicy = indexingPolicy }, - new RequestOptions { OfferEnableRUPerMinuteThroughput = true, OfferThroughput = CloneSettings.TargetMigrationOfferThroughputRUs }); + if (CloneSettings.CopyPartitionKey) + { + // Partition key exists in Source (Unlimited Storage) + newDocumentCollection = (DocumentCollection)await targetClient.CreateDocumentCollectionIfNotExistsAsync + (UriFactory.CreateDatabaseUri(targetDatabaseName), + new DocumentCollection { Id = targetCollectionName, PartitionKey = partitionKeyDefinition, IndexingPolicy = indexingPolicy }, + new RequestOptions { OfferEnableRUPerMinuteThroughput = true, OfferThroughput = CloneSettings.TargetMigrationOfferThroughputRUs }); } else { - newDocumentCollection = (DocumentCollection)await targetClient.CreateDocumentCollectionIfNotExistsAsync - (UriFactory.CreateDatabaseUri(targetDatabaseName), - new DocumentCollection { Id = targetCollectionName, IndexingPolicy = indexingPolicy }, - new RequestOptions { OfferEnableRUPerMinuteThroughput = true, OfferThroughput = CloneSettings.TargetMigrationOfferThroughputRUs }); + newDocumentCollection = (DocumentCollection)await targetClient.CreateDocumentCollectionIfNotExistsAsync + (UriFactory.CreateDatabaseUri(targetDatabaseName), + new DocumentCollection { Id = targetCollectionName, IndexingPolicy = indexingPolicy }, + new RequestOptions { OfferEnableRUPerMinuteThroughput = true, OfferThroughput = CloneSettings.TargetMigrationOfferThroughputRUs }); } } else @@ -292,7 +293,7 @@ public IQueryable GetSourceEntityDocumentQuery(DocumentClient sourceClient } } - public IQueryable GetScrubDataDocumentQuery(DocumentClient targetClient,string filterCondition, int batchSize = -1) + public IQueryable GetScrubDataDocumentQuery(DocumentClient targetClient, string filterCondition, int batchSize = -1) { try { @@ -310,7 +311,7 @@ public IQueryable GetScrubDataDocumentQuery(DocumentClient targetClient,st { scrubDataQuery = "SELECT * FROM c" + " where " + filterCondition; } - + var documentQuery = targetClient.CreateDocumentQuery(UriFactory.CreateDocumentCollectionUri(DatabaseName, CollectionName), scrubDataQuery, queryOptions); return documentQuery; } @@ -325,8 +326,8 @@ public async Task SetTargetRestOfferThroughput() { using (var client = GetTargetDocumentDbClient()) { - var collection = this.GetTargetDocumentCollection(client); - + var collection = this.GetTargetDocumentCollection(client); + FeedOptions queryOptions = new FeedOptions { MaxItemCount = -1, EnableCrossPartitionQuery = true }; //var Ioffer = cosmosClient.CreateOfferQuery(queryOptions); //var offer = Ioffer.AsEnumerable().SingleOrDefault(); @@ -340,7 +341,7 @@ public async Task SetTargetRestOfferThroughput() //.AsEnumerable() //.SingleOrDefault(); } - return true; + return true; } public bool CheckSourceReadability() { @@ -349,7 +350,8 @@ public bool CheckSourceReadability() string topOneRecordQuery = "SELECT TOP 1 * FROM c"; FeedOptions queryOptions = new FeedOptions { MaxItemCount = -1, EnableCrossPartitionQuery = true }; //long sourceTotalRecordCount, targetTotalRecordCount; - try { + try + { using (var cosmosClient = GetSourceDocumentDbClient()) { var document = cosmosClient.CreateDocumentQuery( @@ -359,24 +361,24 @@ public bool CheckSourceReadability() return true; } } - catch(Exception ex) + catch (Exception ex) { logger.LogInfo("Exception during CheckSource Readability"); logger.LogError(ex); } return false; } - + public long GetFilterRecordCount(string filterCondition) { //var TargetCosmosDBSettings = CloneSettings.GetConfigurationSection("SourceCosmosDBSettings"); string targetDatabaseName = CloneSettings.TargetSettings.DatabaseName; - string targetCollectionName = CloneSettings.TargetSettings.CollectionName; + string targetCollectionName = CloneSettings.TargetSettings.CollectionName; string totalCountQuery = "SELECT VALUE COUNT(1) FROM c"; - if(!string.IsNullOrEmpty(filterCondition)) + if (!string.IsNullOrEmpty(filterCondition)) { - totalCountQuery = "SELECT VALUE COUNT(1) FROM c WHERE "+ filterCondition; + totalCountQuery = "SELECT VALUE COUNT(1) FROM c WHERE " + filterCondition; } FeedOptions queryOptions = new FeedOptions { MaxItemCount = -1, EnableCrossPartitionQuery = true }; long totalRecordCount; @@ -414,7 +416,7 @@ public bool CompareRecordCount() UriFactory.CreateDocumentCollectionUri(targetDatabaseName, targetCollectionName), totalCountQuery, queryOptions) .AsEnumerable().First(); } - return (sourceTotalRecordCount == targetTotalRecordCount) ? true: false; + return (sourceTotalRecordCount == targetTotalRecordCount) ? true : false; } public long GetSourceRecordCount() @@ -460,6 +462,6 @@ public long GetRecordCountToScrub(ScrubRule scrubbingRule) } return sourceTotalRecordCount; } - + } } diff --git a/CosmosClone/CosmosCloneCommon/Utility/ObjectScrubber.cs b/CosmosClone/CosmosCloneCommon/Utility/ObjectScrubber.cs index 096b721..f8a1fd5 100644 --- a/CosmosClone/CosmosCloneCommon/Utility/ObjectScrubber.cs +++ b/CosmosClone/CosmosCloneCommon/Utility/ObjectScrubber.cs @@ -4,9 +4,11 @@ using System; using System.Collections.Generic; using System.Linq; -using Newtonsoft.Json.Linq; + using CosmosCloneCommon.Model; +using Newtonsoft.Json.Linq; + namespace CosmosCloneCommon.Utility { public class ObjectScrubber @@ -16,7 +18,7 @@ public List ScrubObjectList(List srcList, ScrubRule scrubRule) //var scrubbedObjects = new List(); var scrubbedObjects = new List(); var propNames = scrubRule.PropertyName.Split('.').ToList(); - if(scrubRule.Type == RuleType.NullValue || scrubRule.Type == RuleType.SingleValue || scrubRule.Type == RuleType.PartialMaskFromLeft || scrubRule.Type == RuleType.PartialMaskFromRight) + if (scrubRule.Type == RuleType.NullValue || scrubRule.Type == RuleType.SingleValue || scrubRule.Type == RuleType.PartialMaskFromLeft || scrubRule.Type == RuleType.PartialMaskFromRight) { foreach (var strObj in srcList) { @@ -25,16 +27,16 @@ public List ScrubObjectList(List srcList, ScrubRule scrubRule) JToken jToken = GetUpdatedJsonArrayValue((JToken)JObject.Parse(strObj), propNames, scrubRule.UpdateValue, scrubRule.Type); scrubbedObjects.Add(jToken); } - catch(Exception ex) + catch (Exception ex) { CloneLogger.LogInfo("Log failed"); CloneLogger.LogError(ex); - throw ; + throw; } - + } } - else if(scrubRule.Type == RuleType.Shuffle) + else if (scrubRule.Type == RuleType.Shuffle) { //get all similar values var propertyValues = new List(); @@ -50,11 +52,11 @@ public List ScrubObjectList(List srcList, ScrubRule scrubRule) { CloneLogger.LogInfo("Log failed"); CloneLogger.LogError(ex); - throw ; + throw; } - + } - + var shuffledTokens = RandomNumberGenerator.Shuffle(propertyValues); var shuffledTokenQ = new Queue(shuffledTokens); @@ -69,7 +71,7 @@ public List ScrubObjectList(List srcList, ScrubRule scrubRule) { CloneLogger.LogInfo("Log failed"); CloneLogger.LogError(ex); - throw ; + throw; } } } @@ -80,13 +82,13 @@ public List ScrubObjectList(List srcList, ScrubRule scrubRule) scrubbedObjects.Add((JToken)strObj); } } - + return scrubbedObjects; } public List GetPropertyValues(JToken token, List propNames, ref List jTokenList) - { - if(jTokenList == null) + { + if (jTokenList == null) { jTokenList = new List(); } @@ -94,7 +96,7 @@ public List GetPropertyValues(JToken token, List propNames, ref bool isLeaflevel = false; - if(propNames.Count > 1) + if (propNames.Count > 1) { if (propNames.Count == 2) isLeaflevel = true; var currentProperty = propNames[1]; @@ -143,7 +145,7 @@ public List GetPropertyValues(JToken token, List propNames, ref } } - } + } return jTokenList; } public JToken GetDocumentShuffledToken(JToken token, List propNames, ref Queue tokenQ) @@ -197,7 +199,7 @@ public JToken GetDocumentShuffledToken(JToken token, List propNames, ref var str3 = jObj.ToString(); jTokenResult = (JToken)jObj; } - } + } if (jTokenResult == null) { jTokenResult = token; @@ -209,7 +211,7 @@ public JToken GetUpdatedJsonArrayValue(JToken token, List propNames, str { if (token == null || token.Type == JTokenType.Null) return null; - JToken jTokenResult=token;//just to initialize + JToken jTokenResult = token;//just to initialize bool isLeaflevel = false; if (propNames.Count > 1) @@ -267,13 +269,13 @@ public JToken GetUpdatedJsonArrayValue(JToken token, List propNames, str } } - if(jTokenResult == null) + if (jTokenResult == null) { jTokenResult = token; } return jTokenResult; } - + private JToken ScrubTokenValue(RuleType? ruleType, JToken tokenToBeScrubbed, string overwriteValue) { if (ruleType.HasValue) diff --git a/CosmosClone/CosmosCloneCommon/app.config b/CosmosClone/CosmosCloneCommon/app.config index aced6ea..ba036b3 100644 --- a/CosmosClone/CosmosCloneCommon/app.config +++ b/CosmosClone/CosmosCloneCommon/app.config @@ -1,31 +1,15 @@ - + - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - - - - \ No newline at end of file + diff --git a/CosmosClone/CosmosCloneCommon/packages.config b/CosmosClone/CosmosCloneCommon/packages.config deleted file mode 100644 index 3bdc108..0000000 --- a/CosmosClone/CosmosCloneCommon/packages.config +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file