Rollback Strategy

Emergency plans and procedures for returning to classic PKI on hybrid problems.

Trigger: One server doesn't work with hybrid certificate.

#!/bin/bash
# rollback-single.sh - Roll back single certificate
 
SERVER="$1"
CERT_NAME="$2"
 
if [ -z "$SERVER" ] || [ -z "$CERT_NAME" ]; then
    echo "Usage: $0 <server> <cert-name>"
    exit 1
fi
 
echo "Rollback for $SERVER/$CERT_NAME"
 
# 1. Backup hybrid certificate
ssh "$SERVER" "cp /etc/ssl/certs/${CERT_NAME}.pem /etc/ssl/certs/${CERT_NAME}.pem.hybrid-backup"
 
# 2. Restore classic certificate (if available)
if [ -f "/backup/certs/${SERVER}/${CERT_NAME}-classic.pem" ]; then
    scp "/backup/certs/${SERVER}/${CERT_NAME}-classic.pem" "$SERVER:/etc/ssl/certs/${CERT_NAME}.pem"
else
    # Reissue with classic CA
    echo "Issuing classic certificate..."
    ssh "$SERVER" "openssl req -new -key /etc/ssl/private/${CERT_NAME}.key -out /tmp/rollback.csr -subj \"/CN=$SERVER\""
    scp "$SERVER:/tmp/rollback.csr" "/tmp/rollback-$SERVER.csr"
 
    openssl ca -config /etc/pki/classic/openssl.cnf \
        -in "/tmp/rollback-$SERVER.csr" \
        -out "/tmp/rollback-$SERVER.pem" \
        -days 365 -batch
 
    scp "/tmp/rollback-$SERVER.pem" "$SERVER:/etc/ssl/certs/${CERT_NAME}.pem"
fi
 
# 3. Reload service
ssh "$SERVER" "systemctl reload nginx || systemctl reload apache2"
 
# 4. Verify
ssh "$SERVER" "openssl s_client -connect localhost:443 -brief" 2>/dev/null | head -5
 
echo "Rollback completed for $SERVER"

Trigger: Multiple servers of a service affected.

#!/bin/bash
# rollback-service.sh - Roll back all servers of a service
 
SERVICE="$1"
SERVERS_FILE="/etc/pki/inventory/${SERVICE}-servers.txt"
 
if [ ! -f "$SERVERS_FILE" ]; then
    echo "Server list not found: $SERVERS_FILE"
    exit 1
fi
 
echo "=== Service Rollback for $SERVICE ==="
echo "Servers: $(wc -l < "$SERVERS_FILE")"
 
# Parallel on all servers
cat "$SERVERS_FILE" | parallel -j 10 '
    echo "Rollback: {}"
    ./rollback-single.sh {} server-cert
'
 
echo "=== Verification ==="
cat "$SERVERS_FILE" | while read server; do
    status=$(curl -sk "https://$server/health" 2>/dev/null && echo "OK" || echo "FAIL")
    echo "$server: $status"
done

Trigger: Hybrid CA certificate has problem.

#!/bin/bash
# rollback-ca.sh - Switch back to classic CA
 
echo "=== CA ROLLBACK ==="
echo "WARNING: This switches the signing CA for all new certificates!"
read -p "Continue? (yes/no): " confirm
[ "$confirm" != "yes" ] && exit 1
 
# 1. Backup current configuration
cp /etc/pki/CA/openssl.cnf /etc/pki/CA/openssl.cnf.hybrid-backup
cp -r /etc/pki/CA/issued /etc/pki/CA/issued-hybrid-backup
 
# 2. Activate classic CA
ln -sf /etc/pki/classic/openssl.cnf /etc/pki/CA/openssl.cnf
export CA_CERT=/etc/pki/classic/intermediate-ca.pem
export CA_KEY=/etc/pki/classic/intermediate-ca.key
 
# 3. Generate new CRL with classic CA
openssl ca -config /etc/pki/CA/openssl.cnf -gencrl -out /var/www/pki/crl.pem
 
# 4. Restart OCSP responder (if present)
systemctl restart ocsp-responder
 
# 5. Documentation
echo "$(date -Iseconds) CA-ROLLBACK: Switch to classic CA" >> /var/log/pki-rollback.log
 
echo "CA rollback completed."
echo "New certificates will now be signed with classic CA."

Trigger: Catastrophic problems with hybrid PKI.

#!/bin/bash
# rollback-complete.sh - Complete rollback to classic PKI
 
echo "========================================"
echo "     COMPLETE PKI ROLLBACK"
echo "========================================"
echo ""
echo "WARNING: This will:"
echo "  1. Revoke all hybrid certificates"
echo "  2. Switch all servers to classic"
echo "  3. Deactivate the hybrid CA"
echo ""
read -p "Enter 'ROLLBACK' to continue: " confirm
[ "$confirm" != "ROLLBACK" ] && exit 1
 
LOG="/var/log/pki-complete-rollback-$(date +%Y%m%d-%H%M%S).log"
exec > >(tee -a "$LOG") 2>&1
 
echo "Start: $(date -Iseconds)"
 
# 1. Identify all hybrid certificates
echo "=== Phase 1: Inventory ==="
find /etc/pki/hybrid/issued -name "*.pem" > /tmp/hybrid-certs.txt
echo "Hybrid certificates: $(wc -l < /tmp/hybrid-certs.txt)"
 
# 2. Create server list
echo "=== Phase 2: Server Mapping ==="
# (Server -> Certificate mapping from CMDB or inventory)
 
# 3. Issue classic certificates for all
echo "=== Phase 3: Classic Re-Issue ==="
while read cert; do
    serial=$(openssl x509 -in "$cert" -serial -noout | cut -d= -f2)
    subject=$(openssl x509 -in "$cert" -subject -noout | sed 's/subject=//')
 
    echo "Re-Issue: $subject"
    # Here: Issue classic certificate
done < /tmp/hybrid-certs.txt
 
# 4. Update servers
echo "=== Phase 4: Deployment ==="
# (Deploy classic certs to all servers in parallel)
 
# 5. Revoke hybrid certificates
echo "=== Phase 5: Revocation ==="
while read cert; do
    openssl ca -config /etc/pki/hybrid/openssl.cnf -revoke "$cert" -crl_reason superseded
done < /tmp/hybrid-certs.txt
 
# 6. Final CRL
openssl ca -config /etc/pki/hybrid/openssl.cnf -gencrl -out /var/www/pki/hybrid-final.crl
 
# 7. Take hybrid CA offline
echo "=== Phase 6: Hybrid CA Deactivation ==="
mv /etc/pki/hybrid /etc/pki/hybrid-DISABLED-$(date +%Y%m%d)
 
echo "=== ROLLBACK COMPLETED ==="
echo "End: $(date -Iseconds)"
echo "Log: $LOG"

# Rollback test script
#!/bin/bash
TEST_SERVER="test-server.example.com"
 
echo "=== Rollback Test $(date) ==="
 
# Save current status
openssl s_client -connect "$TEST_SERVER:443" -brief 2>/dev/null > /tmp/before.txt
 
# Execute rollback
./rollback-single.sh "$TEST_SERVER" server-cert
 
# Check new status
openssl s_client -connect "$TEST_SERVER:443" -brief 2>/dev/null > /tmp/after.txt
 
# Compare
echo "Before:"
grep "Signature" /tmp/before.txt
echo "After:"
grep "Signature" /tmp/after.txt
 
# Restore
echo "Restoring hybrid..."
# ...

Document on every rollback:

• Classic -> Hybrid - Migration
• Disaster Recovery - Larger failures
• Revoke Certificate - Single revocation

« <- Parallel Operation | -> Inventory »

Wolfgang van der Stille @ EMSR DATA d.o.o. - Post-Quantum Cryptography Professional