Skip to content

Upgrade, cleanup and improve #38

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dimitris-norce wants to merge 13 commits into
base: master
Choose a base branch
from
Open

Upgrade, cleanup and improve #38

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
38b8a9d
Upgrade to .NET 4.8, update some nugets
dimitris-norce Apr 11, 2024
099b4ef
Add option to copy more collections
sigdro Apr 11, 2024
ce870a9
Remove binding redirects
dimitris-norce Apr 11, 2024
fc3613a
Upgrade more nugets
dimitris-norce Apr 11, 2024
005672c
ran code cleanup on solution
sigdro Apr 11, 2024
a7f6051
migrate packages.config to package references
dimitris-norce Apr 11, 2024
8b7e99f
Merge branch 'master' of https://github.com/NorceTech/CosmicClone
dimitris-norce Apr 11, 2024
772dd3c
Remove unneeded references from projects
dimitris-norce Apr 11, 2024
16ca2d3
Removed even more unused references
dimitris-norce Apr 11, 2024
8455b36
Remove extra semicolon
dimitris-norce Apr 11, 2024
9947d39
Fix possible nullref exception, improve logging
dimitris-norce Apr 11, 2024
eaa2aea
Bump version to 2.5
dimitris-norce Apr 12, 2024
2dbc51e
Create dependabot.yml
dimitris-norce Oct 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .github/dependabot.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file

version: 2
updates:
- package-ecosystem: "nuget" # See documentation for possible values
directory: "/" # Location of package manifests
schedule:
interval: "weekly"
94 changes: 39 additions & 55 deletions CosmosClone/CloneConsoleRun/App.config
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,74 +1,58 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<configSections>
<section name="SourceCosmosDBSettings" type="System.Configuration.NameValueSectionHandler" />
<section name="TargetCosmosDBSettings" type="System.Configuration.NameValueSectionHandler" />
<section name="SampleCosmosDBSettings" type="System.Configuration.NameValueSectionHandler" />
<section name="SourceCosmosDBSettings" type="System.Configuration.NameValueSectionHandler"/>
<section name="TargetCosmosDBSettings" type="System.Configuration.NameValueSectionHandler"/>
<section name="SampleCosmosDBSettings" type="System.Configuration.NameValueSectionHandler"/>
</configSections>

<SourceCosmosDBSettings>
<add key="EndpointUrl" value="__EndPoint__" />
<add key="AccessKey" value="__Access_" />
<add key="DatabaseName" value="TestDB" />
<add key="CollectionName" value="Sanity1" />
<add key="ReadDelaybetweenRequestsInMs" value="2000" />
<add key="EndpointUrl" value="__EndPoint__"/>
<add key="AccessKey" value="__Access_"/>
<add key="DatabaseName" value="TestDB"/>
<add key="CollectionName" value="Sanity1"/>
<add key="ReadDelaybetweenRequestsInMs" value="2000"/>
</SourceCosmosDBSettings>
<TargetCosmosDBSettings>
<add key="EndpointUrl" value="__EndPoint__" />
<add key="AccessKey" value="__Access_" />
<add key="DatabaseName" value="TestDB" />
<add key="CollectionName" value="Sanity5" />
<add key="OfferThroughputRUs" value="10000" />
<add key="EndpointUrl" value="__EndPoint__"/>
<add key="AccessKey" value="__Access_"/>
<add key="DatabaseName" value="TestDB"/>
<add key="CollectionName" value="Sanity5"/>
<add key="OfferThroughputRUs" value="10000"/>
</TargetCosmosDBSettings>



<SampleCosmosDBSettings>
<add key="EndpointUrl" value="__EndPoint__" />
<add key="AccessKey" value="__Access_" />
<add key="ConnString" value="__connString__" />
<add key="DatabaseName" value="TestDB" />
<add key="IsFixedCollection" value="true" />
<add key="CollectionName" value="Sanity1" />
<add key="OfferThroughputRUs" value="10000" />
<add key="EndpointUrl" value="__EndPoint__"/>
<add key="AccessKey" value="__Access_"/>
<add key="ConnString" value="__connString__"/>
<add key="DatabaseName" value="TestDB"/>
<add key="IsFixedCollection" value="true"/>
<add key="CollectionName" value="Sanity1"/>
<add key="OfferThroughputRUs" value="10000"/>
</SampleCosmosDBSettings>


<appSettings>
<add key="CopyStoredProcedures" value="true" />
<add key="CopyUDFs" value="true" />
<add key="CopyTriggers" value="true" />
<add key="CopyDocuments" value="false" />
<add key="CopyIndexingPolicy" value="true" />
<add key="ReadBatchSize" value="3500" />
<add key="CopyPartitionKey" value="true" />
<add key="WriteBatchSize" value="9000" />
<add key="WriteBatchCount" value="10000" />
<add key="EnableTextLogging" value="true" />
<add key="PopTextFileOnEnd" value="true" />
<add key="SourceOfferThroughputRUs" value="10000" />
<add key="TargetMigrationOfferThroughputRUs" value="10000" />
<add key="TargetRestOfferThroughputRUs" value="1100" />
<add key="ClientSettingsProvider.ServiceUri" value="" />
<add key="ScrubbingRequired" value="true" />
<add key="CopyStoredProcedures" value="true"/>
<add key="CopyUDFs" value="true"/>
<add key="CopyTriggers" value="true"/>
<add key="CopyDocuments" value="false"/>
<add key="CopyIndexingPolicy" value="true"/>
<add key="ReadBatchSize" value="3500"/>
<add key="CopyPartitionKey" value="true"/>
<add key="WriteBatchSize" value="9000"/>
<add key="WriteBatchCount" value="10000"/>
<add key="EnableTextLogging" value="true"/>
<add key="PopTextFileOnEnd" value="true"/>
<add key="SourceOfferThroughputRUs" value="10000"/>
<add key="TargetMigrationOfferThroughputRUs" value="10000"/>
<add key="TargetRestOfferThroughputRUs" value="1100"/>
<add key="ClientSettingsProvider.ServiceUri" value=""/>
<add key="ScrubbingRequired" value="true"/>
</appSettings>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6.1" />
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.8"/>
</startup>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="Microsoft.Azure.Documents.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.22.0.0" newVersion="1.22.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-10.0.0.0" newVersion="10.0.0.0" />
</dependentAssembly>
<dependentAssembly>
<assemblyIdentity name="System.Buffers" publicKeyToken="cc7b13ffcd2ddd51" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-4.0.3.0" newVersion="4.0.3.0" />
</dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>
</configuration>
62 changes: 19 additions & 43 deletions CosmosClone/CloneConsoleRun/CloneConsoleRun.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,12 @@
<OutputType>Exe</OutputType>
<RootNamespace>CloneConsoleRun</RootNamespace>
<AssemblyName>CloneConsoleRun</AssemblyName>
<TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
<TargetFrameworkVersion>v4.8</TargetFrameworkVersion>
<FileAlignment>512</FileAlignment>
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
<NuGetPackageImportStamp>
</NuGetPackageImportStamp>
<TargetFrameworkProfile />
</PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
<PlatformTarget>AnyCPU</PlatformTarget>
Expand All @@ -34,42 +35,8 @@
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
<Reference Include="Microsoft.Azure.CosmosDB.BulkImport, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.Azure.CosmosDB.BulkExecutor.1.2.0\lib\net451\Microsoft.Azure.CosmosDB.BulkImport.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Azure.Documents.Client, Version=2.1.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>..\packages\Microsoft.Azure.DocumentDB.2.1.3\lib\net461\Microsoft.Azure.Documents.Client.dll</HintPath>
</Reference>
<Reference Include="MongoDB.Bson, Version=2.4.4.0, Culture=neutral, processorArchitecture=MSIL">
<HintPath>..\packages\MongoDB.Bson.2.4.4\lib\net45\MongoDB.Bson.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=10.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<HintPath>..\packages\Newtonsoft.Json.10.0.2\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
<Reference Include="System" />
<Reference Include="System.Buffers, Version=4.0.3.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
<HintPath>..\packages\System.Buffers.4.5.0\lib\netstandard2.0\System.Buffers.dll</HintPath>
</Reference>
<Reference Include="System.Core" />
<Reference Include="System.Memory, Version=4.0.1.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL">
<HintPath>..\packages\System.Memory.4.5.0\lib\netstandard2.0\System.Memory.dll</HintPath>
</Reference>
<Reference Include="System.Net.Http.WinHttpHandler, Version=4.0.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Net.Http.WinHttpHandler.4.5.0\lib\net461\System.Net.Http.WinHttpHandler.dll</HintPath>
</Reference>
<Reference Include="System.Numerics" />
<Reference Include="System.Numerics.Vectors, Version=4.1.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Numerics.Vectors.4.4.0\lib\net46\System.Numerics.Vectors.dll</HintPath>
</Reference>
<Reference Include="System.Runtime.CompilerServices.Unsafe, Version=4.0.4.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
<HintPath>..\packages\System.Runtime.CompilerServices.Unsafe.4.5.0\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll</HintPath>
</Reference>
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Data" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
<Compile Include="Program.cs" />
Expand All @@ -81,20 +48,29 @@
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\CosmosCloneCommon\CosmosCloneCommon.csproj">
<Project>{523e1cc4-0286-498c-b184-3413e10d61de}</Project>
<Name>CosmosCloneCommon</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Azure.CosmosDB.BulkExecutor">
<Version>1.8.9</Version>
</PackageReference>
<PackageReference Include="MongoDB.Bson">
<Version>2.24.0</Version>
</PackageReference>
<PackageReference Include="MongoDB.Bson.signed">
<Version>2.19.0</Version>
</PackageReference>
<PackageReference Include="Newtonsoft.Json">
<Version>13.0.3</Version>
</PackageReference>
<PackageReference Include="System.Runtime.CompilerServices.Unsafe">
<Version>6.0.0</Version>
</PackageReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="..\packages\Microsoft.Azure.DocumentDB.2.1.3\build\Microsoft.Azure.DocumentDB.targets" Condition="Exists('..\packages\Microsoft.Azure.DocumentDB.2.1.3\build\Microsoft.Azure.DocumentDB.targets')" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\Microsoft.Azure.DocumentDB.2.1.3\build\Microsoft.Azure.DocumentDB.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Microsoft.Azure.DocumentDB.2.1.3\build\Microsoft.Azure.DocumentDB.targets'))" />
</Target>
</Project>
12 changes: 4 additions & 8 deletions CosmosClone/CloneConsoleRun/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,11 @@

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using CosmosCloneCommon;

using CosmosCloneCommon.Migrator;
using CosmosCloneCommon.Model;
using CosmosCloneCommon.Utility;

using logger = CosmosCloneCommon.Utility.CloneLogger;
using Newtonsoft.Json;

namespace CloneConsoleRun
{
Expand Down Expand Up @@ -72,8 +68,8 @@ public static void TestCosmosScrubbing()
var documentMigrator = new CosmosCloneCommon.Migrator.DocumentMigrator();
documentMigrator.StartCopy(scrubRules).Wait();
//var result = tcs.StartScrub(scrubRules);
}
}



}
}
1 change: 0 additions & 1 deletion CosmosClone/CloneConsoleRun/Properties/AssemblyInfo.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;

// General Information about an assembly is controlled through the following
Expand Down
34 changes: 18 additions & 16 deletions CosmosClone/CloneConsoleRun/Sample/CosmosSampleDBHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,14 @@ namespace CloneConsoleRun.Sample
{
using System;
using System.Threading.Tasks;

using CosmosCloneCommon.Utility;

using Microsoft.Azure.Documents;
using Microsoft.Azure.Documents.Client;

public class CosmosSampleDBHelper
{
{
private ConnectionPolicy ConnectionPolicy;
public CosmosSampleDBHelper()
{
Expand All @@ -25,7 +27,7 @@ public CosmosSampleDBHelper()
this.ConnectionPolicy.RetryOptions.MaxRetryAttemptsOnThrottledRequests = 20;
this.ConnectionPolicy.RetryOptions.MaxRetryWaitTimeInSeconds = 600;
}

public DocumentClient GetSampleDocumentDbClient()
{
try
Expand Down Expand Up @@ -68,7 +70,7 @@ public async Task<DocumentCollection> CreateSampleDocumentCollection(DocumentCli
//no partition key if it is a fixed collection
newDocumentCollection = (DocumentCollection)await sampleClient.CreateDocumentCollectionIfNotExistsAsync
(UriFactory.CreateDatabaseUri(sampleDatabaseName),
new DocumentCollection { Id = sampleCollectionName},
new DocumentCollection { Id = sampleCollectionName },
new RequestOptions { OfferThroughput = offerThroughput });
}
return newDocumentCollection;
Expand All @@ -90,22 +92,22 @@ public async Task<DocumentCollection> CreateTargetDocumentCollection(DocumentCli

await targetClient.CreateDatabaseIfNotExistsAsync(new Database { Id = targetDatabaseName });
DocumentCollection newDocumentCollection;
if (partitionKeyDefinition != null && partitionKeyDefinition.Paths.Count>0)
if (partitionKeyDefinition != null && partitionKeyDefinition.Paths.Count > 0)
{
if(CloneSettings.CopyPartitionKey)
{
// Partition key exists in Source (Unlimited Storage)
newDocumentCollection = (DocumentCollection)await targetClient.CreateDocumentCollectionIfNotExistsAsync
(UriFactory.CreateDatabaseUri(targetDatabaseName),
new DocumentCollection { Id = targetCollectionName, PartitionKey = partitionKeyDefinition, IndexingPolicy = indexingPolicy },
new RequestOptions { OfferEnableRUPerMinuteThroughput = true, OfferThroughput = CloneSettings.TargetMigrationOfferThroughputRUs });
if (CloneSettings.CopyPartitionKey)
{
// Partition key exists in Source (Unlimited Storage)
newDocumentCollection = (DocumentCollection)await targetClient.CreateDocumentCollectionIfNotExistsAsync
(UriFactory.CreateDatabaseUri(targetDatabaseName),
new DocumentCollection { Id = targetCollectionName, PartitionKey = partitionKeyDefinition, IndexingPolicy = indexingPolicy },
new RequestOptions { OfferEnableRUPerMinuteThroughput = true, OfferThroughput = CloneSettings.TargetMigrationOfferThroughputRUs });
}
else
{
newDocumentCollection = (DocumentCollection)await targetClient.CreateDocumentCollectionIfNotExistsAsync
(UriFactory.CreateDatabaseUri(targetDatabaseName),
new DocumentCollection { Id = targetCollectionName, IndexingPolicy = indexingPolicy },
new RequestOptions { OfferEnableRUPerMinuteThroughput = true, OfferThroughput = CloneSettings.TargetMigrationOfferThroughputRUs });
newDocumentCollection = (DocumentCollection)await targetClient.CreateDocumentCollectionIfNotExistsAsync
(UriFactory.CreateDatabaseUri(targetDatabaseName),
new DocumentCollection { Id = targetCollectionName, IndexingPolicy = indexingPolicy },
new RequestOptions { OfferEnableRUPerMinuteThroughput = true, OfferThroughput = CloneSettings.TargetMigrationOfferThroughputRUs });
}
}
else
Expand All @@ -124,6 +126,6 @@ public async Task<DocumentCollection> CreateTargetDocumentCollection(DocumentCli
throw;
}
}

}
}
5 changes: 3 additions & 2 deletions CosmosClone/CloneConsoleRun/Sample/Entity.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ namespace CloneConsoleRun.Sample
{
using System;
using System.Collections.Generic;

using CosmosCloneCommon.Model;

public class Entity
Expand Down Expand Up @@ -99,9 +100,9 @@ public static Entitytest getRandomTestEntity()
testEntity.Words.Add("rex123");
return testEntity;
}
}
}






Expand Down
7 changes: 4 additions & 3 deletions CosmosClone/CloneConsoleRun/Sample/EntityV2.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,11 @@ namespace CloneConsoleRun.Sample
{
using System;
using System.Collections.Generic;

using CosmosCloneCommon.Model;

public class EntityV2
{
{
public string FullName { get; set; }
public string Description { get; set; }
public Boolean IsActive { get; set; }
Expand All @@ -30,7 +31,7 @@ public static EntityV2 getRandomEntity()
entity.Description = "Test Sample Description " + entity.SuperId.ToString();
entity.EntityType = RandomNumberGenerator.GetRandomEntityType();
var employeeid = RandomNumberGenerator.GetNext();
entity.EmailAddress = "intialTest"+ employeeid .ToString()+ "@test.com";
entity.EmailAddress = "intialTest" + employeeid.ToString() + "@test.com";
entity.PhoneNumber = RandomNumberGenerator.GetNext().ToString();
entity.IsActive = true;
entity.ModifiedTime = DateTime.UtcNow;
Expand All @@ -46,7 +47,7 @@ public static EntityV2 getRandomEntity()
}






Expand Down
Loading