Distributed systems essentials

// Cosmos DB: read from the closest region first, fail over in order
var cosmos = new CosmosClient(endpoint, new DefaultAzureCredential(), new CosmosClientOptions
{
    ApplicationPreferredRegions = [Regions.AustraliaEast, Regions.AustraliaSoutheast]
});

// SQL Server: a second connection string for replica reads
// "Server=tcp:myag-listener;Database=Bank;ApplicationIntent=ReadOnly;..."
builder.Services.AddDbContext<BankDb>(o => o.UseSqlServer(primaryCs));
builder.Services.AddDbContext<BankReadDb>(o => o.UseSqlServer(readOnlyCs)
    .UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking));

// Cassandra: RF = 3 per data centre. LOCAL_QUORUM on both sides → 2 + 2 > 3
var cluster = Cluster.Builder()
    .AddContactPoint("cassandra-1")
    .WithQueryOptions(new QueryOptions().SetConsistencyLevel(ConsistencyLevel.LocalQuorum))
    .Build();
var session = await cluster.ConnectAsync("payments");

var insert = new SimpleStatement(
    "INSERT INTO balances (account_id, balance) VALUES (?, ?)", accountId, 50m);
await session.ExecuteAsync(insert);                 // waits for 2 of 3 local replicas

var read = new SimpleStatement("SELECT balance FROM balances WHERE account_id = ?", accountId)
    .SetConsistencyLevel(ConsistencyLevel.One);     // cheaper, may be stale
var rows = await session.ExecuteAsync(read);

// Leader election with an Azure Blob lease: only the lease holder runs the job
var lease = container.GetBlobClient("leader/statement-run").GetBlobLeaseClient();
while (!ct.IsCancellationRequested)
{
    try
    {
        await lease.AcquireAsync(TimeSpan.FromSeconds(30), cancellationToken: ct);
        using var renew = new PeriodicTimer(TimeSpan.FromSeconds(10));
        var work = RunAsLeaderAsync(ct);                     // the singleton job
        while (!work.IsCompleted && await renew.WaitForNextTickAsync(ct))
            await lease.RenewAsync(cancellationToken: ct);   // keep leadership
        await work;
        await lease.ReleaseAsync(cancellationToken: ct);
    }
    catch (RequestFailedException e) when (e.Status == 409) // someone else leads
    {
        await Task.Delay(TimeSpan.FromSeconds(15), ct);
    }
}

If a renew fails, stop working immediately: another instance may already be leader. Pair the lease with a fencing check on the data you write.

// One instance runs the job; a fencing check protects the data anyway
var @lock = new SqlDistributedLock("statement-run-2026-10", connectionString);
await using var handle = await @lock.TryAcquireAsync(TimeSpan.Zero, ct);
if (handle is null) return;                        // another pod has it

long token = await jobs.NextFencingTokenAsync(ct);  // e.g. a SQL SEQUENCE
foreach (var batch in await LoadBatchesAsync(ct))
{
    // Only write if no newer holder has written already
    int updated = await db.StatementRuns
        .Where(r => r.Id == batch.RunId && r.LastToken < token)
        .ExecuteUpdateAsync(s => s.SetProperty(r => r.LastToken, token)
                                  .SetProperty(r => r.Status, "Done"), ct);
    if (updated == 0) break;                       // we've been fenced off
}

// Works anywhere: one database, one local transaction
using (var scope = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled))
{
    await accounts.DebitAsync(from, amount, ct);
    await ledger.PostAsync(from, to, amount, ct);   // same DB / connection string
    scope.Complete();
}

// Two different databases → escalates to MSDTC (Windows only). Avoid in new designs.
if (OperatingSystem.IsWindows())
    TransactionManager.ImplicitDistributedTransactions = true;

MSDTC escalation doesn't work in Linux containers, and it ties you to Windows hosts. In the cloud, design for sagas + outbox from day one.

// A thread-safe Lamport clock
public sealed class LamportClock
{
    private long _time;

    // Local event or just before sending a message
    public long Tick() => Interlocked.Increment(ref _time);

    // On receiving a message stamped with the sender's time
    public long Receive(long remote)
    {
        while (true)
        {
            long current = Interlocked.Read(ref _time);
            long next = Math.Max(current, remote) + 1;
            if (Interlocked.CompareExchange(ref _time, next, current) == current)
                return next;
        }
    }
}