From a3a3fcab409793dcfefc65b884a3420516ba17c9 Mon Sep 17 00:00:00 2001 From: Kosta Petan Date: Mon, 19 Feb 2024 21:22:13 +0000 Subject: [PATCH] eventhubs streaming --- .devcontainer/Dockerfile | 10 +-- .devcontainer/devcontainer.json | 3 +- src/apps/WorkflowsApp/WorkflowsApp.csproj | 2 +- src/apps/cli/cli.csproj | 2 +- src/apps/gh-flow-df/gh-flow-df.csproj | 2 +- src/apps/gh-flow/Program.cs | 74 +++++++++++++------ .../Services/GithubWebHookProcessor.cs | 2 +- src/apps/gh-flow/gh-flow.csproj | 11 +-- src/apps/seed-memory/seed-memory.csproj | 2 +- .../Elsa.SemanticKernel.csproj | 2 +- .../Microsoft.AI.DevTeam.Skills.csproj | 2 +- .../Microsoft.AI.DevTeam.csproj | 12 +-- 12 files changed, 73 insertions(+), 51 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 3aacb89db..cb7c250a4 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,11 +1,5 @@ -FROM mcr.microsoft.com/devcontainers/dotnet:0-7.0 +FROM mcr.microsoft.com/devcontainers/dotnet:8.0 # Install the xz-utils package RUN apt-get update && apt-get install -y xz-utils ca-certificates curl gnupg -RUN curl -fsSL https://aka.ms/install-azd.sh | bash - -RUN curl -s https://ngrok-agent.s3.amazonaws.com/ngrok.asc | \ - sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null && \ - echo "deb https://ngrok-agent.s3.amazonaws.com buster main" | \ - sudo tee /etc/apt/sources.list.d/ngrok.list && \ - sudo apt update && sudo apt install ngrok \ No newline at end of file +RUN curl -fsSL https://aka.ms/install-azd.sh | bash \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c0a345e21..7a9468127 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -18,8 +18,7 @@ }, "ghcr.io/azure/azure-dev/azd:0": { "version": "stable" - }, - "ghcr.io/jlaundry/devcontainer-features/azure-functions-core-tools:1": {} + } }, "postCreateCommand": "bash .devcontainer/startup.sh", "hostRequirements": { diff --git a/src/apps/WorkflowsApp/WorkflowsApp.csproj b/src/apps/WorkflowsApp/WorkflowsApp.csproj index 413793f84..91a86e335 100644 --- a/src/apps/WorkflowsApp/WorkflowsApp.csproj +++ b/src/apps/WorkflowsApp/WorkflowsApp.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 enable enable diff --git a/src/apps/cli/cli.csproj b/src/apps/cli/cli.csproj index b930ed06f..5edeb9bc6 100644 --- a/src/apps/cli/cli.csproj +++ b/src/apps/cli/cli.csproj @@ -2,7 +2,7 @@ Exe - net7.0 + net8.0 enable enable diff --git a/src/apps/gh-flow-df/gh-flow-df.csproj b/src/apps/gh-flow-df/gh-flow-df.csproj index 07d73342f..892e46dcf 100644 --- a/src/apps/gh-flow-df/gh-flow-df.csproj +++ b/src/apps/gh-flow-df/gh-flow-df.csproj @@ -6,7 +6,7 @@ - net7.0 + net8.0 v4 Exe diff --git a/src/apps/gh-flow/Program.cs b/src/apps/gh-flow/Program.cs index 19db7a65d..c08ecb94a 100644 --- a/src/apps/gh-flow/Program.cs +++ b/src/apps/gh-flow/Program.cs @@ -63,19 +63,27 @@ builder.Services.AddSingleton(); builder.Host.UseOrleans(siloBuilder => { - + if (builder.Environment.IsDevelopment()) { var connectionString = builder.Configuration.GetValue("AzureOptions:CosmosConnectionString"); - siloBuilder.AddMemoryStreams("StreamProvider") - .AddMemoryGrainStorage("PubSubStore");; - siloBuilder.UseCosmosReminderService( o => + //siloBuilder.AddMemoryStreams("StreamProvider"); + siloBuilder.UseCosmosReminderService(o => { - o.ConfigureCosmosClient(connectionString); - o.ContainerName = "reminders"; - o.DatabaseName = "devteam"; - o.IsResourceCreationEnabled = true; + o.ConfigureCosmosClient(connectionString); + o.ContainerName = "reminders"; + o.DatabaseName = "devteam"; + o.IsResourceCreationEnabled = true; }); + siloBuilder.AddCosmosGrainStorage( + name: "PubSubStore", + configureOptions: o => + { + o.ConfigureCosmosClient(connectionString); + o.ContainerName = "pubsubstore"; + o.DatabaseName = "pubsub"; + o.IsResourceCreationEnabled = true; + }); siloBuilder.AddCosmosGrainStorage( name: "messages", configureOptions: o => @@ -85,7 +93,26 @@ builder.Host.UseOrleans(siloBuilder => o.DatabaseName = "devteam"; o.IsResourceCreationEnabled = true; }); - siloBuilder.UseLocalhostClustering(); + var ehConnectionString = builder.Configuration.GetValue("AzureOptions:EventHubsConnectionString"); + var storageConnectionString = builder.Configuration.GetValue("AzureOptions:StorageAccountConnectionString"); + + siloBuilder.UseLocalhostClustering() + .AddEventHubStreams("StreamProvider", (ISiloEventHubStreamConfigurator configurator) => + { + configurator.ConfigureEventHub(builder => builder.Configure(options => + { + options.ConfigureEventHubConnection( + ehConnectionString, + "sk-dev-team", + "$Default"); + })); + configurator.UseAzureTableCheckpointer( + builder => builder.Configure(options => + { + options.ConfigureTableServiceClient(storageConnectionString); + options.PersistInterval = TimeSpan.FromSeconds(10); + })); + }); ; } else { @@ -100,24 +127,24 @@ builder.Host.UseOrleans(siloBuilder => options.ResponseTimeout = TimeSpan.FromMinutes(3); options.SystemResponseTimeout = TimeSpan.FromMinutes(3); }); - siloBuilder.Configure(options => - { - options.ResponseTimeout = TimeSpan.FromMinutes(3); - }); - siloBuilder.UseCosmosClustering( o => + siloBuilder.Configure(options => + { + options.ResponseTimeout = TimeSpan.FromMinutes(3); + }); + siloBuilder.UseCosmosClustering(o => { o.ConfigureCosmosClient(cosmosDbconnectionString); o.ContainerName = "devteam"; o.DatabaseName = "clustering"; o.IsResourceCreationEnabled = true; }); - - siloBuilder.UseCosmosReminderService( o => + + siloBuilder.UseCosmosReminderService(o => { - o.ConfigureCosmosClient(cosmosDbconnectionString); - o.ContainerName = "devteam"; - o.DatabaseName = "reminders"; - o.IsResourceCreationEnabled = true; + o.ConfigureCosmosClient(cosmosDbconnectionString); + o.ContainerName = "devteam"; + o.DatabaseName = "reminders"; + o.IsResourceCreationEnabled = true; }); siloBuilder.AddCosmosGrainStorage( name: "messages", @@ -130,8 +157,8 @@ builder.Host.UseOrleans(siloBuilder => }); //TODO: Add streaming here - } - + } + }); builder.Services.Configure(options => @@ -189,7 +216,8 @@ static IKernel CreateKernel(IServiceProvider provider) return new KernelBuilder() .WithLoggerFactory(loggerFactory) .WithAzureChatCompletionService(openAiConfig.DeploymentOrModelId, openAIClient) - .WithRetryBasic(new BasicRetryConfig { + .WithRetryBasic(new BasicRetryConfig + { MaxRetryCount = 5, UseExponentialBackoff = true }).Build(); diff --git a/src/apps/gh-flow/Services/GithubWebHookProcessor.cs b/src/apps/gh-flow/Services/GithubWebHookProcessor.cs index 376fe14dc..b3e82062f 100644 --- a/src/apps/gh-flow/Services/GithubWebHookProcessor.cs +++ b/src/apps/gh-flow/Services/GithubWebHookProcessor.cs @@ -74,7 +74,7 @@ public sealed class GithubWebHookProcessor : WebhookEventProcessor await HandleNewAsk(issueNumber, skillName, functionName, suffix, input, org, repo); } } - catch (System.Exception) + catch (System.Exception ex) { _logger.LogError("Processing issue comment event"); } diff --git a/src/apps/gh-flow/gh-flow.csproj b/src/apps/gh-flow/gh-flow.csproj index cc7e902d1..d2120dcb9 100644 --- a/src/apps/gh-flow/gh-flow.csproj +++ b/src/apps/gh-flow/gh-flow.csproj @@ -5,7 +5,7 @@ - net7.0 + net8.0 enable enable true @@ -16,12 +16,13 @@ - - - + + + - + + diff --git a/src/apps/seed-memory/seed-memory.csproj b/src/apps/seed-memory/seed-memory.csproj index e73fad29e..fd33f7cca 100644 --- a/src/apps/seed-memory/seed-memory.csproj +++ b/src/apps/seed-memory/seed-memory.csproj @@ -2,7 +2,7 @@ Exe - net7.0 + net8.0 waf_import enable enable diff --git a/src/libs/Elsa.SemanticKernel/Elsa.SemanticKernel.csproj b/src/libs/Elsa.SemanticKernel/Elsa.SemanticKernel.csproj index 85445d758..c74693d35 100644 --- a/src/libs/Elsa.SemanticKernel/Elsa.SemanticKernel.csproj +++ b/src/libs/Elsa.SemanticKernel/Elsa.SemanticKernel.csproj @@ -1,7 +1,7 @@ - net7.0 + net8.0 Activities for calling Semantic Kernel SDK diff --git a/src/libs/Microsoft.AI.DevTeam.Skills/Microsoft.AI.DevTeam.Skills.csproj b/src/libs/Microsoft.AI.DevTeam.Skills/Microsoft.AI.DevTeam.Skills.csproj index fb2eeb6a7..d54d803d5 100644 --- a/src/libs/Microsoft.AI.DevTeam.Skills/Microsoft.AI.DevTeam.Skills.csproj +++ b/src/libs/Microsoft.AI.DevTeam.Skills/Microsoft.AI.DevTeam.Skills.csproj @@ -1,7 +1,7 @@  - net7.0 + net8.0 enable enable diff --git a/src/libs/Microsoft.AI.DevTeam/Microsoft.AI.DevTeam.csproj b/src/libs/Microsoft.AI.DevTeam/Microsoft.AI.DevTeam.csproj index be95cfbdb..ea740822b 100644 --- a/src/libs/Microsoft.AI.DevTeam/Microsoft.AI.DevTeam.csproj +++ b/src/libs/Microsoft.AI.DevTeam/Microsoft.AI.DevTeam.csproj @@ -1,15 +1,15 @@  - net7.0 + net8.0 enable enable - - - + + + @@ -17,8 +17,8 @@ - - + +