[PM-8108] Add Duo SDK v4 metadata to Duo Two Factor Provider (#4774)

* Migrate Duo Two Factor Configuration to support both v2 and v4

* Postgres Migrations

* SQLite migrations

* comment updates for SQLite; Query changes for consistency;

* comment clean up; formatting

util/Migrator/DbScripts/2024-09-05_00_SyncDuoVersionFourMetaDataToVersionTwo.sql

@@ -0,0 +1,81 @@
+-- Create temporary table to store User IDs
+CREATE TABLE #TempUserIDs (
+    RowNum INT IDENTITY(1,1) PRIMARY KEY,
+    ID UNIQUEIDENTIFIER
+);
+
+-- Populate temporary table with User IDs
+INSERT INTO #TempUserIDs (ID)
+SELECT Id
+FROM [dbo].[User]
+WHERE TwoFactorProviders LIKE '%"2":%'
+  AND ISJSON(TwoFactorProviders) = 1;
+
+DECLARE @UserBatchSize INT = 1000;
+DECLARE @TotalUserRows INT = (SELECT COUNT(*) FROM #TempUserIDs);
+DECLARE @UserBatchNum INT = 0;
+
+WHILE @UserBatchNum * @UserBatchSize < @TotalUserRows
+BEGIN
+    -- Update Users
+    UPDATE U
+    SET TwoFactorProviders = JSON_MODIFY(
+        JSON_MODIFY(
+            U.TwoFactorProviders,
+            '$."2".MetaData.ClientSecret',
+            JSON_VALUE(U.TwoFactorProviders, '$."2".MetaData.SKey')
+        ),
+        '$."2".MetaData.ClientId',
+        JSON_VALUE(U.TwoFactorProviders, '$."2".MetaData.IKey')
+    )
+    FROM [dbo].[User] U
+    INNER JOIN #TempUserIDs T ON U.Id = T.ID
+    WHERE T.RowNum > @UserBatchNum * @UserBatchSize
+      AND T.RowNum <= (@UserBatchNum + 1) * @UserBatchSize;
+
+    SET @UserBatchNum = @UserBatchNum + 1;
+END
+
+-- Clean up
+DROP TABLE #TempUserIDs;
+
+-- Create temporary table to store Organization IDs
+CREATE TABLE #TempOrganizationIDs (
+    RowNum INT IDENTITY(1,1) PRIMARY KEY,
+    ID UNIQUEIDENTIFIER
+);
+
+-- Populate temporary table with Organization IDs
+INSERT INTO #TempOrganizationIDs (ID)
+SELECT Id
+FROM [dbo].[Organization]
+WHERE TwoFactorProviders LIKE '%"6":%'
+  AND ISJSON(TwoFactorProviders) = 1;
+
+DECLARE @OrganizationBatchSize INT = 1000;
+DECLARE @TotalOrganizationRows INT = (SELECT COUNT(*) FROM #TempOrganizationIDs);
+DECLARE @OrganizationBatchNum INT = 0;
+
+WHILE @OrganizationBatchNum * @OrganizationBatchSize < @TotalOrganizationRows
+BEGIN
+    -- Update Organizations
+    UPDATE Org
+    SET TwoFactorProviders = JSON_MODIFY(
+        JSON_MODIFY(
+            Org.TwoFactorProviders,
+            '$."6".MetaData.ClientSecret',
+            JSON_VALUE(Org.TwoFactorProviders, '$."6".MetaData.SKey')
+        ),
+        '$."6".MetaData.ClientId',
+        JSON_VALUE(Org.TwoFactorProviders, '$."6".MetaData.IKey')
+    )
+    FROM [dbo].[Organization] Org
+    INNER JOIN #TempOrganizationIDs T ON Org.Id = T.ID
+    WHERE T.RowNum > @OrganizationBatchNum * @OrganizationBatchSize
+      AND T.RowNum <= (@OrganizationBatchNum + 1) * @OrganizationBatchSize;
+
+    SET @OrganizationBatchNum = @OrganizationBatchNum + 1;
+END
+
+-- Clean up
+DROP TABLE #TempOrganizationIDs;

util/MySqlMigrations/HelperScripts/2024-09-05_00_SyncDuoVersionFourMetadataToVersionTwo.sql

@@ -0,0 +1,29 @@
+/* Update User Table */
+UPDATE
+	`User` U
+SET
+	U.TwoFactorProviders = JSON_SET(
+        JSON_SET(
+            U.TwoFactorProviders, '$."2".MetaData.ClientSecret',
+	JSON_UNQUOTE(U.TwoFactorProviders ->'$."2".MetaData.SKey')),
+	'$."2".MetaData.ClientId',
+	JSON_UNQUOTE(U.TwoFactorProviders -> '$."2".MetaData.IKey'))
+WHERE
+	JSON_CONTAINS(TwoFactorProviders,
+	'{"2":{}}')
+	AND JSON_VALID(TwoFactorProviders);
+
+/* Update Organization Table */
+UPDATE
+	Organization o
+SET
+	o.TwoFactorProviders = JSON_SET(
+        JSON_SET(
+            o.TwoFactorProviders, '$."6".MetaData.ClientSecret',
+	JSON_UNQUOTE(o.TwoFactorProviders ->'$."6".MetaData.SKey')),
+	'$."6".MetaData.ClientId',
+	JSON_UNQUOTE(o.TwoFactorProviders -> '$."6".MetaData.IKey'))
+WHERE
+	JSON_CONTAINS(o.TwoFactorProviders,
+	'{"6":{}}')
+	AND JSON_VALID(o.TwoFactorProviders);

util/MySqlMigrations/Migrations/20240909181805_GenerateDuoSDKVersion4TwoFactorMetadata.cs

@@ -0,0 +1,22 @@
+using Bit.Core.Utilities;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Bit.MySqlMigrations.Migrations;
+
+/// <inheritdoc />
+public partial class GenerateDuoSDKVersion4TwoFactorMetadata : Migration
+{
+    private const string _duoTwoFactorDataMigrationsScript = "MySqlMigrations.HelperScripts.2024-09-05_00_SyncDuoVersionFourMetadataToVersionTwo.sql";
+
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql(CoreHelpers.GetEmbeddedResourceContentsAsync(_duoTwoFactorDataMigrationsScript));
+    }
+
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        throw new Exception("Irreversible migration");
+    }
+}

util/MySqlMigrations/MySqlMigrations.csproj

@@ -31,5 +31,6 @@
     <EmbeddedResource Include="HelperScripts\2022-03-01_00_Down_MigrateOrganizationApiKeys.sql" />
     <EmbeddedResource Include="HelperScripts\2024-04-25_00_EnableOrgsCollectionEnhancements.sql" />
     <EmbeddedResource Include="HelperScripts\2024-08-26_00_FinalFlexibleCollectionsDataMigrations.sql" />
+    <EmbeddedResource Include="HelperScripts\2024-09-05_00_SyncDuoVersionFourMetadataToVersionTwo.sql" />
   </ItemGroup>
 </Project>

util/PostgresMigrations/HelperScripts/2024-09-05_00_SyncDuoVersionFourMetadataToVersionTwo.psql

@@ -0,0 +1,28 @@
+-- Update User table
+update
+	"User"
+set
+	"TwoFactorProviders" = jsonb_set(
+                jsonb_set("TwoFactorProviders"::jsonb,
+	'{2,MetaData,ClientSecret}',
+	("TwoFactorProviders"::jsonb -> '2' -> 'MetaData' -> 'SKey')),
+	'{2,MetaData,ClientId}',
+	("TwoFactorProviders"::jsonb -> '2' -> 'MetaData' -> 'IKey'))
+where
+	"TwoFactorProviders" like '%"2":%'
+	and jsonb_typeof("TwoFactorProviders"::jsonb) = 'object';
+
+-- Update Organization table
+update
+	"Organization"
+set
+	"TwoFactorProviders" = jsonb_set(
+                jsonb_set("TwoFactorProviders"::jsonb,
+	'{6,MetaData,ClientSecret}',
+	("TwoFactorProviders"::jsonb -> '6' -> 'MetaData' -> 'SKey')),
+	'{6,MetaData,ClientId}',
+	("TwoFactorProviders"::jsonb -> '6' -> 'MetaData' -> 'IKey'))
+where
+	"TwoFactorProviders" like '%"6":%'
+	and jsonb_typeof("TwoFactorProviders"::jsonb) = 'object';
+	

util/PostgresMigrations/Migrations/20240909181749_GenerateDuoSDKVersion4TwoFactorMetadata.cs

@@ -0,0 +1,23 @@
+using Bit.Core.Utilities;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Bit.PostgresMigrations.Migrations;
+
+public partial class GenerateDuoSDKVersion4TwoFactorMetadata : Migration
+{
+    private const string _scriptLocation =
+    "PostgresMigrations.HelperScripts.2024-09-05_00_SyncDuoVersionFourMetadataToVersionTwo.psql";
+
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql(CoreHelpers.GetEmbeddedResourceContentsAsync(_scriptLocation));
+    }
+
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        // the changes here are additive and not destructive by adding the v4 data we are not impacting application function.
+        // there is no meaningful impact to the application with this migration.
+    }
+}

util/PostgresMigrations/PostgresMigrations.csproj

@@ -26,5 +26,6 @@
     <EmbeddedResource Include="HelperScripts\2022-03-01_00_Down_MigrateOrganizationApiKeys.psql" />
     <EmbeddedResource Include="HelperScripts\2024-04-25_00_EnableOrgsCollectionEnhancements.psql" />
     <EmbeddedResource Include="HelperScripts\2024-08-26_00_FinalFlexibleCollectionsDataMigrations.psql" />
+    <EmbeddedResource Include="HelperScripts\2024-09-05_00_SyncDuoVersionFourMetadataToVersionTwo.psql" />
   </ItemGroup>
 </Project>

util/SqliteMigrations/HelperScripts/2024-09-05_00_SyncDuoVersionFourMetadataToVersionTwo.sql

@@ -0,0 +1,35 @@
+/*
+I spent some time looking into how to batch process these queries, but SQLite does not have
+a good interface for batch processing. So to improve readability and maintain veloicty I've opted 
+for a single update query for each table: "User" and "Organization".
+
+Part of the Reasoning is that SQLite, while not a "toy database", is not usually used in larger
+deployments. Scalability is difficult in SQLite, so the assumption is the database is small for 
+installations using SQLite. So not running these in a batch should not impact on users who do 
+use SQLite.
+*/
+
+-- Update User accounts
+UPDATE "User"
+SET TwoFactorProviders = json_set(
+    json_set("User".TwoFactorProviders,
+        '$."2".MetaData.ClientSecret',
+        json_extract("User".TwoFactorProviders, '$."2".MetaData.SKey')),
+        '$."2".MetaData.ClientId',
+        json_extract("User".TwoFactorProviders, '$."2".MetaData.IKey')
+)
+WHERE TwoFactorProviders LIKE '%"2":%'
+    AND JSON_VALID(TwoFactorProviders) = 1;
+
+-- Update Organizations
+UPDATE "Organization"
+SET TwoFactorProviders = json_set(
+    json_set("Organization".TwoFactorProviders,
+        '$."6".MetaData.ClientSecret',
+        json_extract("Organization".TwoFactorProviders, '$."6".MetaData.SKey')),
+        '$."6".MetaData.ClientId',
+        json_extract("Organization".TwoFactorProviders, '$."6".MetaData.IKey')
+)
+WHERE TwoFactorProviders LIKE '%"6":%'
+    AND JSON_VALID(TwoFactorProviders) = 1;
+    

util/SqliteMigrations/Migrations/20240909181758_GenerateDuoSDKVersion4TwoFactorMetadata.cs

@@ -0,0 +1,22 @@
+using Bit.Core.Utilities;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Bit.SqliteMigrations.Migrations;
+
+public partial class GenerateDuoSDKVersion4TwoFactorMetadata : Migration
+{
+    private const string _duoTwoFactorDataMigrationsScript = "SqliteMigrations.HelperScripts.2024-09-05_00_SyncDuoVersionFourMetadataToVersionTwo.sql";
+
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql(CoreHelpers.GetEmbeddedResourceContentsAsync(_duoTwoFactorDataMigrationsScript));
+    }
+
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        // the changes here are additive and not destructive by adding the v4 data we are not impacting application function.
+        // there is no meaningful impact to the application with this migration.
+    }
+}

util/SqliteMigrations/SqliteMigrations.csproj

@@ -26,6 +26,7 @@
     <EmbeddedResource Include="HelperScripts\2023-12-04_00_Down_GrantIndexes.sql" />
     <EmbeddedResource Include="HelperScripts\2024-04-25_00_EnableOrgsCollectionEnhancements.sql" />
     <EmbeddedResource Include="HelperScripts\2024-08-26_00_FinalFlexibleCollectionsDataMigrations.sql" />
+    <EmbeddedResource Include="HelperScripts\2024-09-05_00_SyncDuoVersionFourMetadataToVersionTwo.sql" />
   </ItemGroup>
 
 </Project>