Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 | import { Heartbeat } from '@/types/api/heartbeats';
import { TypedUpdateInput } from '@/types/backend/dynamo';
import {
TABLE_STATUS, typedScan, typedUpdate
} from '@/utils/backend/dynamoTyped';
import { sendAlertMessage } from '@/utils/backend/texts';
import { getLogger } from '@/utils/common/logger';
const logger = getLogger('status');
// The amount of time to wait for a heartbeat before failing over (in ms)
const maxSpacing = 5 * 60 * 1000;
export async function main() {
logger.trace('main', ...arguments);
// Get all of the heartbeats
const heartbeatsScan = await typedScan<Heartbeat>({
TableName: TABLE_STATUS,
});
const heartbeats = heartbeatsScan.Items || [];
const now = Date.now();
const changedHeartbeats = heartbeats
.filter(hb => (
hb.IsFailed &&
now - (hb.LastHeartbeat || 0) <= maxSpacing
) || (
!hb.IsFailed &&
now - (hb.LastHeartbeat || 0) >= maxSpacing
));
const updateDynamoPromises = Promise.all(changedHeartbeats.map(hb => {
hb.IsFailed = !hb.IsFailed;
const updateConfig: TypedUpdateInput<Heartbeat> = {
TableName: TABLE_STATUS,
Key: {
Server: hb.Server,
},
ExpressionAttributeNames: {
'#IsFailed': 'IsFailed',
},
ExpressionAttributeValues: {
':IsFailed': hb.IsFailed,
},
UpdateExpression: 'SET #IsFailed = :IsFailed',
};
// Set active to false if the heartbeat failed
if (hb.IsFailed) {
updateConfig.ExpressionAttributeValues = updateConfig.ExpressionAttributeValues || {};
updateConfig.ExpressionAttributeNames['#IsActive'] = 'IsActive';
updateConfig.ExpressionAttributeValues[':IsActive'] = false;
updateConfig.UpdateExpression += ', #IsActive = :IsActive';
}
return typedUpdate(updateConfig);
}));
const messages = changedHeartbeats
.map(hb => {
const programCaps = 'VHF';
const primaryHeartbeats = heartbeats.filter(hb2 => hb2.IsPrimary);
const secondaryHeartbeats = heartbeats.filter(hb2 => !hb2.IsPrimary);
const parts = {
changed: `${hb.IsPrimary ? 'Primary' : 'Secondary'} ${programCaps} server (${hb.Server})`,
all: `All ${programCaps} servers (${heartbeats.map(hb2 => hb2.Server).join(', ')})`,
primary: `primary ${programCaps} server (${primaryHeartbeats.map(hb2 => hb2.Server).join(', ')})`,
secondary: `secondary ${programCaps} server (${secondaryHeartbeats.map(hb2 => hb2.Server).join(', ')})`,
};
const primaryUp = primaryHeartbeats
.filter(hb2 => !hb2.IsFailed)
.length > 0;
const secondaryUp = secondaryHeartbeats
.filter(hb2 => !hb2.IsFailed)
.length > 0;
const isSecondary = secondaryHeartbeats.length > 0;
if (primaryUp && secondaryUp) {
if (hb.IsPrimary) {
return `${parts.changed} is back online. Switching back to ${hb.Server}.`;
} else {
return `${parts.changed} is back online. ${parts.all} are online.`;
}
} else if (!primaryUp && secondaryUp) {
if (hb.IsPrimary) {
return `${parts.changed} is down. Switching to ${parts.secondary}.`;
} else {
return `${parts.changed} is back online. Switching to ${parts.secondary}, ${parts.primary} is still offline.`;
}
} else if (primaryUp && !isSecondary) {
return `${parts.changed} is back online. ${programCaps} recording now occuring.`;
} else if (primaryUp && !secondaryUp) {
if (hb.IsPrimary) {
return `${parts.changed} is back online. Switching to ${parts.primary}, ${parts.secondary} is still offline.`;
} else {
return `${parts.changed} is offline. Continuing to record ${programCaps} on ${parts.primary}.`;
}
} else if (!primaryUp && (!secondaryUp || !isSecondary)) {
if (hb.IsPrimary) {
return `${parts.changed} is offline. ${programCaps} recording is no longer occuring${isSecondary ? ` because ${parts.secondary} is still offline` : ''}.`;
} else {
return `${parts.changed} is offline. ${programCaps} recording is no longer occuring because ${parts.primary} is still offline.`;
}
} else {
return `Unkown state. IsPrimary: ${hb.IsPrimary}, primaryUp: ${primaryUp}, secondaryUp: ${secondaryUp}, isSecondary: ${isSecondary}. MEOW.`;
}
});
if (messages.length > 0) {
logger.debug('main', 'alert messages', messages);
await sendAlertMessage('Vhf', messages.join('\n'));
}
await updateDynamoPromises;
}
|