Scalability, availability & fault tolerance

var credential = new DefaultAzureCredential();
builder.Services.AddDataProtection()
    .SetApplicationName("payments-api")                    // same name on every instance
    .PersistKeysToAzureBlobStorage(new Uri(keysBlobUri), credential)
    .ProtectKeysWithAzureKeyVault(new Uri(keyVaultKeyId), credential);

builder.Services.AddStackExchangeRedisCache(o =>
    o.Configuration = builder.Configuration.GetConnectionString("redis"));
builder.Services.AddSession(o => o.IdleTimeout = TimeSpan.FromMinutes(20)); // stored in Redis

var app = builder.Build();
app.UseSession();

// Program.cs
builder.Services.AddReverseProxy().LoadFromConfig(builder.Configuration.GetSection("ReverseProxy"));
app.MapReverseProxy();

// appsettings.json
"ReverseProxy": {
  "Routes":   { "api": { "ClusterId": "api", "Match": { "Path": "/api/{**catch-all}" } } },
  "Clusters": {
    "api": {
      "LoadBalancingPolicy": "LeastRequests",
      "HealthCheck": {
        "Active":  { "Enabled": true, "Interval": "00:00:10", "Path": "/health/ready" },
        "Passive": { "Enabled": true }
      },
      "Destinations": {
        "a": { "Address": "https://10.0.1.4/" },
        "b": { "Address": "https://10.0.1.5/" }
      }
    }
  }
}

public sealed class HashRing
{
    private readonly ulong[] _positions;   // sorted points on the ring
    private readonly string[] _nodes;      // node that owns each point

    public HashRing(IEnumerable<string> nodes, int virtualNodes = 100)
    {
        var ring = new SortedList<ulong, string>();
        foreach (var node in nodes)
            for (var i = 0; i < virtualNodes; i++)
                ring[Hash($"{node}#{i}")] = node;   // many points per node = even spread
        _positions = [.. ring.Keys];
        _nodes = [.. ring.Values];
    }

    public string GetNode(string key)
    {
        var i = Array.BinarySearch(_positions, Hash(key));
        if (i < 0) i = ~i;                                // first point clockwise
        return _nodes[i == _positions.Length ? 0 : i];    // wrap around the ring
    }

    private static ulong Hash(string s) => XxHash3.HashToUInt64(Encoding.UTF8.GetBytes(s));
}

builder.Services.AddHealthChecks()
    .AddCheck("self", () => HealthCheckResult.Healthy(), tags: ["live"])
    .AddSqlServer(builder.Configuration.GetConnectionString("sql")!, tags: ["ready"])   // AspNetCore.HealthChecks.SqlServer
    .AddDbContextCheck<AppDbContext>(tags: ["ready"]);   // ...HealthChecks.EntityFrameworkCore

builder.Services.AddDbContext<AppDbContext>(o =>
    o.UseSqlServer(builder.Configuration.GetConnectionString("sql"),
        sql => sql.EnableRetryOnFailure()));   // retries transient errors

app.MapHealthChecks("/health/live",  new() { Predicate = c => c.Tags.Contains("live") });
app.MapHealthChecks("/health/ready", new() { Predicate = c => c.Tags.Contains("ready") });

Keep liveness checks shallow (is the process alive?). If liveness checks the database, a DB outage makes the platform restart every instance at once.

// HttpClient: one line gives timeout, retry, circuit breaker and bulkhead
builder.Services.AddHttpClient<FxClient>(c => c.BaseAddress = new("https://fx.internal/"))
    .AddStandardResilienceHandler(o =>
    {
        o.AttemptTimeout.Timeout = TimeSpan.FromSeconds(2);
        o.TotalRequestTimeout.Timeout = TimeSpan.FromSeconds(10);
        o.Retry.MaxRetryAttempts = 3;               // backoff + jitter by default
    });

// Anything else (DB, SDK): a named Polly v8 pipeline. First added = outermost
builder.Services.AddResiliencePipeline("ledger", p => p
    .AddRetry(new() { MaxRetryAttempts = 3, BackoffType = DelayBackoffType.Exponential, UseJitter = true })
    .AddCircuitBreaker(new() { FailureRatio = 0.5, MinimumThroughput = 10, BreakDuration = TimeSpan.FromSeconds(30) })
    .AddTimeout(TimeSpan.FromSeconds(3)));

// Usage (inject ResiliencePipelineProvider<string> provider)
await provider.GetPipeline("ledger").ExecuteAsync(async t => await ledger.PostAsync(entry, t), ct);

Only retry idempotent operations, or send an idempotency key. Retrying a payment POST without one can charge the customer twice. The standard handler retries every method by default; call o.Retry.DisableForUnsafeHttpMethods() to stop that.

// Cosmos DB: nearest region first, automatic move to the next if it fails
var cosmos = new CosmosClient(endpoint, new DefaultAzureCredential(), new CosmosClientOptions
{
    ApplicationPreferredRegions = [Regions.AustraliaEast, Regions.AustraliaSoutheast]
});

[MemoryDiagnoser]                                   // also shows bytes allocated per call
public class ParseBench
{
    private readonly string _line = "2026-10-16,ACC123,150.25";

    [Benchmark(Baseline = true)]
    public decimal WithSplit() => decimal.Parse(_line.Split(',')[2]);   // allocates 3 strings

    [Benchmark]
    public decimal WithSpan()
    {
        var s = _line.AsSpan();
        return decimal.Parse(s[(s.LastIndexOf(',') + 1)..]);            // no allocations
    }
}
// Run in Release: BenchmarkRunner.Run<ParseBench>();

Measure first. Find the real bottleneck with dotnet-counters / dotnet-trace (often the DB or a chatty remote call) before tuning C# code.

// Track in-flight work (L) and completed work (λ) for a queue worker
public sealed class WorkerMetrics
{
    private readonly UpDownCounter<long> _inFlight;   // L
    private readonly Counter<long> _done;             // its rate = λ

    public WorkerMetrics(IMeterFactory meterFactory)
    {
        var meter = meterFactory.Create("Bank.Payments.Worker");
        _inFlight = meter.CreateUpDownCounter<long>("payments.in_flight");
        _done = meter.CreateCounter<long>("payments.processed");
    }

    public async Task TrackAsync(Func<Task> work)
    {
        _inFlight.Add(1);
        try { await work(); _done.Add(1); }
        finally { _inFlight.Add(-1); }
    }
}
// dotnet-counters monitor -n PaymentsWorker --counters Bank.Payments.Worker

builder.Services.AddOpenTelemetry()
    .UseAzureMonitor()                                 // Azure.Monitor.OpenTelemetry.AspNetCore
    .WithMetrics(m => m.AddMeter("Bank.Payments"));    // plus our custom meter

public sealed class PaymentSli
{
    private readonly Counter<long> _total;
    private readonly Counter<long> _good;

    public PaymentSli(IMeterFactory meterFactory)
    {
        var meter = meterFactory.Create("Bank.Payments");
        _total = meter.CreateCounter<long>("payments.sli.total");
        _good  = meter.CreateCounter<long>("payments.sli.good");
    }

    public void Record(bool succeeded, TimeSpan settleTime)
    {
        _total.Add(1);
        if (succeeded && settleTime < TimeSpan.FromSeconds(5)) _good.Add(1);   // SLI = good / total
    }
}

The SLO maths (good / total over 30 days, burn-rate alerts) lives in the monitoring tool (KQL, PromQL), not in app code. The app's job is to emit honest metrics.

// Cosmos: the account default is Session; relax it for one read where latency matters more
var options = new ItemRequestOptions { ConsistencyLevel = ConsistencyLevel.Eventual };
var item = await container.ReadItemAsync<Quote>(id, new PartitionKey(pair), options);

// EF Core: optimistic concurrency with a SQL Server rowversion column
public class Account
{
    public int Id { get; set; }
    public decimal Balance { get; set; }
    [Timestamp] public byte[] RowVersion { get; set; } = [];
}

// SQL: always connect to the failover-group listener, never a server name
var sql = "Server=tcp:payments-fog.database.windows.net;Database=Payments;" +
          "Authentication=Active Directory Default;ConnectRetryCount=3";

// Blobs: read from the secondary region if the primary is down (RA-GRS)
var blobs = new BlobServiceClient(new Uri("https://paymentsdocs.blob.core.windows.net"),
    new DefaultAzureCredential(),
    new BlobClientOptions
    {
        GeoRedundantSecondaryUri = new Uri("https://paymentsdocs-secondary.blob.core.windows.net")
    });

Test it. A DR plan that has never been run in a drill usually misses its RTO because something was forgotten: DNS caches, firewall rules, secrets or connection strings in the second region.

// YARP: the edge sets X-Cell from a tenant lookup; each route sends one cell to its cluster
"Routes": {
  "cell-1": { "ClusterId": "cell-1",
              "Match": { "Path": "{**catch-all}", "Headers": [ { "Name": "X-Cell", "Values": [ "cell-1" ] } ] } },
  "cell-2": { "ClusterId": "cell-2",
              "Match": { "Path": "{**catch-all}", "Headers": [ { "Name": "X-Cell", "Values": [ "cell-2" ] } ] } }
}

// Cosmos DB: nearest region first; last-writer-wins on a property you control
var cosmos = new CosmosClient(endpoint, new DefaultAzureCredential(), new CosmosClientOptions
{
    ApplicationPreferredRegions = [Regions.AustraliaEast, Regions.AustraliaSoutheast]
});
await database.CreateContainerIfNotExistsAsync(new ContainerProperties("profiles", "/customerId")
{
    ConflictResolutionPolicy = new ConflictResolutionPolicy
    {
        Mode = ConflictResolutionMode.LastWriterWins,
        ResolutionPath = "/updatedAtTicks"
    }
});

Last writer wins is fine for a customer profile, not for a balance. For money, give each account one home region for writes, so there is nothing to resolve.