init
This commit is contained in:
@@ -0,0 +1,625 @@
|
||||
import config from "../config.ts";
|
||||
import axios from "axios";
|
||||
import { redis, redisCount } from "../utils/redis.ts";
|
||||
import { postgres as pg } from "../utils/postgres.ts";
|
||||
import type { PoolConfig, PoolRegion } from "./utils.ts";
|
||||
import type { Client } from "pg";
|
||||
|
||||
const incrInterval = 5 * 1000;
|
||||
const decrInterval = 15 * 1000;
|
||||
const cleanupInterval = 5 * 60 * 1000;
|
||||
|
||||
// If postgres isn't configured we can still run in stateless mode
|
||||
// Only start/get/terminate can be used, otherwise exception will be thrown
|
||||
const postgres = !pg ? (null as unknown as Client) : pg;
|
||||
|
||||
export abstract class VMManager {
|
||||
protected isLarge = false;
|
||||
protected region: PoolRegion = "US";
|
||||
private limitSize = 0;
|
||||
private minSize = 0;
|
||||
protected hostname: string | undefined;
|
||||
|
||||
constructor({ isLarge, region, limitSize, minSize, hostname }: PoolConfig) {
|
||||
this.isLarge = isLarge;
|
||||
this.region = region;
|
||||
this.limitSize = Number(limitSize) || 0;
|
||||
this.minSize = Number(minSize) || 0;
|
||||
this.hostname = hostname;
|
||||
}
|
||||
|
||||
public getIsLarge = () => {
|
||||
return this.isLarge;
|
||||
};
|
||||
|
||||
public getRegion = () => {
|
||||
return this.region;
|
||||
};
|
||||
|
||||
public getMinSize = () => {
|
||||
return this.minSize;
|
||||
};
|
||||
|
||||
public getLimitSize = () => {
|
||||
return this.limitSize;
|
||||
};
|
||||
|
||||
public getTargetBuffer = () => {
|
||||
let buffer = this.limitSize * 0.02;
|
||||
// If ramping config, adjust buffer based on the hour
|
||||
// During ramp down hours, keep a smaller buffer
|
||||
// During ramp up hours, keep a larger buffer
|
||||
// const rampDownHours = config.VM_POOL_RAMP_DOWN_HOURS.split(",").map(Number);
|
||||
// const rampUpHours = config.VM_POOL_RAMP_UP_HOURS.split(",").map(Number);
|
||||
// const nowHour = new Date().getUTCHours();
|
||||
// const isRampDown =
|
||||
// rampDownHours.length &&
|
||||
// pointInInterval24(nowHour, rampDownHours[0], rampDownHours[1]);
|
||||
// const isRampUp =
|
||||
// rampUpHours.length &&
|
||||
// pointInInterval24(nowHour, rampUpHours[0], rampUpHours[1]);
|
||||
// if (isRampDown) {
|
||||
// buffer *= 0.5;
|
||||
// } else if (isRampUp) {
|
||||
// buffer *= 1.5;
|
||||
// }
|
||||
return Math.ceil(buffer);
|
||||
};
|
||||
|
||||
public getCurrentSize = async () => {
|
||||
const { rows } = await postgres.query(
|
||||
`SELECT count(1) FROM vbrowser WHERE pool = $1`,
|
||||
[this.getPoolName()],
|
||||
);
|
||||
return Number(rows[0]?.count);
|
||||
};
|
||||
|
||||
public getPoolName = () => {
|
||||
return this.id + (this.isLarge ? "Large" : "") + this.region;
|
||||
};
|
||||
|
||||
public getAvailableCount = async (): Promise<number> => {
|
||||
const { rows } = await postgres.query(
|
||||
`SELECT count(1) FROM vbrowser WHERE pool = $1 and state = 'available'`,
|
||||
[this.getPoolName()],
|
||||
);
|
||||
return Number(rows[0]?.count);
|
||||
};
|
||||
|
||||
public getStagingCount = async (): Promise<number> => {
|
||||
const { rows } = await postgres.query(
|
||||
`SELECT count(1) FROM vbrowser WHERE pool = $1 and state = 'staging'`,
|
||||
[this.getPoolName()],
|
||||
);
|
||||
return Number(rows[0]?.count);
|
||||
};
|
||||
|
||||
public getAvailableVBrowsers = async (): Promise<string[]> => {
|
||||
const { rows } = await postgres.query(
|
||||
`SELECT vmid from vbrowser WHERE pool = $1 and state = 'available'`,
|
||||
[this.getPoolName()],
|
||||
);
|
||||
return rows.map((row: any) => row.vmid);
|
||||
};
|
||||
|
||||
public getStagingVBrowsers = async (): Promise<string[]> => {
|
||||
const { rows } = await postgres.query(
|
||||
`SELECT vmid from vbrowser WHERE pool = $1 and state = 'staging'`,
|
||||
[this.getPoolName()],
|
||||
);
|
||||
return rows.map((row: any) => row.vmid);
|
||||
};
|
||||
|
||||
public getTag = () => {
|
||||
return (
|
||||
(config.VBROWSER_TAG || "vbrowser") +
|
||||
this.region +
|
||||
(this.isLarge ? "Large" : "")
|
||||
);
|
||||
};
|
||||
|
||||
public assignVM = async (
|
||||
roomId: string,
|
||||
uid: string,
|
||||
): Promise<AssignedVM | undefined> => {
|
||||
if (!roomId || !uid) {
|
||||
return undefined;
|
||||
}
|
||||
// Update and use SKIP LOCKED to ensure each consumer gets a different one
|
||||
const { rows } = await postgres.query(
|
||||
`
|
||||
UPDATE vbrowser
|
||||
SET "roomId" = $1, uid = $2, "heartbeatTime" = NOW(), "assignTime" = NOW(), state = 'used'
|
||||
WHERE id = (
|
||||
SELECT id
|
||||
FROM vbrowser
|
||||
WHERE state = 'available'
|
||||
AND pool = $3
|
||||
ORDER BY "creationTime" DESC
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
)
|
||||
RETURNING data, pass`,
|
||||
[roomId, uid, this.getPoolName()],
|
||||
);
|
||||
let vm: VM | undefined = rows[0]?.data;
|
||||
let pass: string = rows[0]?.pass;
|
||||
if (!vm) {
|
||||
return;
|
||||
}
|
||||
return { ...vm, pass, assignTime: Date.now() };
|
||||
};
|
||||
|
||||
public resetVM = async (vmid: string, roomId?: string): Promise<void> => {
|
||||
if (roomId !== undefined) {
|
||||
// verify the roomId matches if user initiated
|
||||
const { rows } = await postgres.query(
|
||||
`SELECT "roomId" FROM vbrowser WHERE pool = $1 AND vmid = $2`,
|
||||
[this.getPoolName(), vmid],
|
||||
);
|
||||
if (rows[0]?.roomId && rows[0]?.roomId !== roomId) {
|
||||
console.log(
|
||||
"[RESET] %s: roomId mismatch on %s, expected %s, got %s",
|
||||
this.getPoolName(),
|
||||
vmid,
|
||||
rows[0]?.roomId,
|
||||
roomId,
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.log("[RESET]", this.getPoolName(), vmid, roomId);
|
||||
// We generally want to reuse if the provider has per-hour billing
|
||||
// Since most user sessions are less than an hour
|
||||
// Otherwise if it's per-second or Docker, it's easier to just terminate it on reboot
|
||||
if (this.reuseVMs) {
|
||||
// To reset without API calls/reimaging (logic can be reused on any cloud provider so we can avoid individual reboot implementations):
|
||||
// Update vbrowser script to generate uuid password
|
||||
// add admin endpoints to vbrowser to get password and reboot
|
||||
// to reset, hit reboot endpoint with admin key OR call cloud reboot method
|
||||
// in checkstaging, check password endpoint with admin key, update DB once available
|
||||
const { rows } = await postgres.query(
|
||||
`SELECT image FROM vbrowser WHERE pool = $1 AND vmid = $2`,
|
||||
[this.getPoolName(), vmid],
|
||||
);
|
||||
const vmImageId = rows[0]?.image;
|
||||
// Check if VM needs to be reimaged
|
||||
if (this.imageId !== vmImageId) {
|
||||
await this.reimageVM(vmid);
|
||||
redisCount("vBrowserReimage");
|
||||
// Update the vmImageId
|
||||
await postgres.query(
|
||||
`UPDATE vbrowser SET image = $3 WHERE pool = $1 AND vmid = $2`,
|
||||
[this.getPoolName(), vmid, this.imageId],
|
||||
);
|
||||
} else {
|
||||
await this.rebootVM(vmid);
|
||||
}
|
||||
// we could crash here and then row will remain in used state
|
||||
// Once the heartbeat becomes stale cleanup will reset it again
|
||||
const result = await postgres.query(
|
||||
`
|
||||
INSERT INTO vbrowser(pool, vmid, "creationTime", state)
|
||||
VALUES($1, $2, NOW(), 'staging')
|
||||
ON CONFLICT(pool, vmid) DO
|
||||
UPDATE SET state = 'staging',
|
||||
"roomId" = NULL, uid = NULL, retries = 0, "heartbeatTime" = NULL, "assignTime" = NULL, data = NULL
|
||||
`,
|
||||
[this.getPoolName(), vmid],
|
||||
);
|
||||
console.log("UPSERT", result.rowCount);
|
||||
// Normally this should be an update, but we could insert if:
|
||||
// if cleaning up a VM we didn't record in db on create
|
||||
// if we resized down and deleted db row but didn't complete the termination
|
||||
} else {
|
||||
this.terminateVMWrapper(vmid);
|
||||
}
|
||||
};
|
||||
|
||||
public startVMWrapper = async () => {
|
||||
// generate credentials and boot a VM
|
||||
const password = crypto.randomUUID();
|
||||
const id = await this.startVM(password);
|
||||
// We might fail to record it if crashing here but cleanup will reset it
|
||||
await postgres.query(
|
||||
`
|
||||
INSERT INTO vbrowser(pool, vmid, "creationTime", state, image)
|
||||
VALUES($1, $2, NOW(), 'staging', $3)`,
|
||||
[this.getPoolName(), id, this.imageId],
|
||||
);
|
||||
redisCount("vBrowserLaunches");
|
||||
return id;
|
||||
};
|
||||
|
||||
protected terminateVMWrapper = async (vmid: string) => {
|
||||
console.log("[TERMINATE]", this.getPoolName(), vmid);
|
||||
// Update the DB before calling terminate
|
||||
// If we don't actually complete the termination, cleanup will reset it
|
||||
const { command, rowCount } = await postgres.query(
|
||||
`DELETE FROM vbrowser WHERE pool = $1 AND vmid = $2 RETURNING id`,
|
||||
[this.getPoolName(), vmid],
|
||||
);
|
||||
console.log(command, rowCount);
|
||||
// We can log the VM lifetime by returning the creationTime and diffing
|
||||
await this.terminateVM(vmid);
|
||||
};
|
||||
|
||||
checkVMReady = async (host: string) => {
|
||||
// NOTE: This URL doesn't work for docker since it doesn't have /
|
||||
// But since we don't reuse docker VMs it's ok
|
||||
const url = "https://" + host.replace("/", "/health");
|
||||
try {
|
||||
// const out = execSync(`curl -i -L -v --ipv4 '${host}'`);
|
||||
// if (!out.toString().startsWith('OK') && !out.toString().startsWith('404 page not found')) {
|
||||
// throw new Error('mismatched response from health');
|
||||
// }
|
||||
const resp = await axios({
|
||||
method: "GET",
|
||||
url,
|
||||
timeout: 1000,
|
||||
});
|
||||
// Check to make sure the VM was recently rebooted (we could also check on the password to ensure reset)
|
||||
const timeSinceBoot = Date.now() / 1000 - Number(resp.data);
|
||||
// console.log(timeSinceBoot);
|
||||
return this.reuseVMs ? timeSinceBoot < 150 * 1000 : true;
|
||||
} catch (e) {
|
||||
// console.log(url, e.message, e.response?.status);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
public runBackgroundJobs = async () => {
|
||||
const resizeVMGroupIncr = async () => {
|
||||
const availableCount = await this.getAvailableCount();
|
||||
const stagingCount = await this.getStagingCount();
|
||||
const currentSize = await this.getCurrentSize();
|
||||
let launch = false;
|
||||
launch =
|
||||
availableCount + stagingCount < this.getTargetBuffer() &&
|
||||
currentSize < (this.getLimitSize() || Infinity);
|
||||
if (launch) {
|
||||
console.log(
|
||||
"[RESIZE-INCR]",
|
||||
this.getPoolName(),
|
||||
"target:",
|
||||
this.getTargetBuffer(),
|
||||
"available:",
|
||||
availableCount,
|
||||
"staging:",
|
||||
stagingCount,
|
||||
"currentSize:",
|
||||
currentSize,
|
||||
"limit:",
|
||||
this.getLimitSize(),
|
||||
);
|
||||
try {
|
||||
await this.startVMWrapper();
|
||||
} catch (e: any) {
|
||||
console.log(
|
||||
e.response?.status,
|
||||
JSON.stringify(e.response?.data),
|
||||
e.config?.url,
|
||||
e.config?.data,
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const resizeVMGroupDecr = async () => {
|
||||
const availableCount = await this.getAvailableCount();
|
||||
const stagingCount = await this.getStagingCount();
|
||||
let unlaunch = false;
|
||||
unlaunch = availableCount + stagingCount > this.getTargetBuffer();
|
||||
if (unlaunch) {
|
||||
// use SKIP LOCKED to delete to avoid deleting VM that might be assigning
|
||||
// filter to only VMs eligible for deletion
|
||||
// they must be up for long enough
|
||||
// keep the oldest min pool size number of VMs
|
||||
// Hetzner/DO/Scaleway rounds up to nearest hour
|
||||
let modulo = 3600;
|
||||
const { rows } = await postgres.query(
|
||||
`
|
||||
DELETE FROM vbrowser
|
||||
WHERE id = (
|
||||
SELECT id
|
||||
FROM vbrowser
|
||||
WHERE pool = $1
|
||||
AND state = 'available'
|
||||
AND id >= (SELECT id from vbrowser WHERE pool = $1 ORDER BY id ASC LIMIT 1 OFFSET $2)
|
||||
AND CAST(extract(epoch from now() - "creationTime") as INT) % $3 > $4
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 1
|
||||
) RETURNING vmid`,
|
||||
[
|
||||
this.getPoolName(),
|
||||
this.getMinSize(),
|
||||
modulo,
|
||||
config.VM_MIN_UPTIME_MINUTES * 60, // to seconds
|
||||
],
|
||||
);
|
||||
const first = rows[0];
|
||||
if (first) {
|
||||
console.log("[RESIZE-DECR] %s: %s", this.getPoolName(), first.vmid);
|
||||
await this.terminateVMWrapper(first.vmid);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const cleanupVMGroup = async () => {
|
||||
// Reset hanging VMs
|
||||
// It's possible we created a VM but lost track of it
|
||||
// Take the list of VMs from API
|
||||
// subtract VMs that have a heartbeat or available or staging
|
||||
let allVMs = [];
|
||||
try {
|
||||
allVMs = await this.listVMs(this.getTag());
|
||||
} catch (e) {
|
||||
console.log(
|
||||
"[CLEANUP] %s: failed to fetch VM list",
|
||||
this.getPoolName(),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const { rows } = await postgres.query(
|
||||
`
|
||||
SELECT vmid from vbrowser
|
||||
WHERE pool = $1
|
||||
AND
|
||||
("heartbeatTime" > (NOW() - INTERVAL '5 minutes')
|
||||
OR state = 'staging'
|
||||
OR state = 'available')
|
||||
`,
|
||||
[this.getPoolName()],
|
||||
);
|
||||
const inUse = new Set(rows.map((row: any) => row.vmid));
|
||||
console.log(
|
||||
"[CLEANUP] %s: found %s VMs, %s to keep",
|
||||
this.getPoolName(),
|
||||
allVMs.length,
|
||||
inUse.size,
|
||||
);
|
||||
for (let server of allVMs) {
|
||||
if (!inUse.has(server.id)) {
|
||||
redisCount("vBrowserCleanup");
|
||||
console.log("[CLEANUP]", this.getPoolName(), server.id);
|
||||
try {
|
||||
await this.resetVM(server.id);
|
||||
//this.terminateVMWrapper(server.id);
|
||||
} catch (e: any) {
|
||||
console.warn("[CLEANUP]", this.getPoolName(), e.response?.data);
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const checkStaging = async () => {
|
||||
// Increment retry count and return data
|
||||
const { rows } = await postgres.query(
|
||||
`
|
||||
UPDATE vbrowser
|
||||
SET retries = retries + 1
|
||||
WHERE pool = $1 and state = 'staging'
|
||||
RETURNING id, vmid, data, retries
|
||||
`,
|
||||
[this.getPoolName()],
|
||||
);
|
||||
const stagingPromises = rows.map(async (row: any): Promise<string> => {
|
||||
const rowid = row.id;
|
||||
const vmid: string = row.vmid;
|
||||
const retryCount: number = row.retries;
|
||||
let vm: VM | null = row.data;
|
||||
if (retryCount < this.minRetries) {
|
||||
if (config.NODE_ENV === "development") {
|
||||
console.log(
|
||||
"[CHECKSTAGING] %s: [vmid: %s] [attempt: %s] waiting for minRetries",
|
||||
this.getPoolName(),
|
||||
vmid,
|
||||
retryCount,
|
||||
);
|
||||
}
|
||||
// Do a minimum # of retries to give reboot time
|
||||
return [vmid, retryCount, false].join(",");
|
||||
}
|
||||
// Fetch data on first attempt
|
||||
// Try again only every once in a while to reduce load on API
|
||||
const shouldFetchVM =
|
||||
retryCount === this.minRetries + 1 || retryCount % 30 === 0;
|
||||
// Refetch the VM every once in a while even if cached to check if it should be deleted
|
||||
if (shouldFetchVM) {
|
||||
try {
|
||||
vm = await this.getVM(vmid);
|
||||
} catch (e: any) {
|
||||
console.warn(e.response?.data);
|
||||
if (e.response?.status === 404) {
|
||||
// Remove the VM because the provider says it doesn't exist
|
||||
await postgres.query("DELETE FROM vbrowser WHERE id = $1", [
|
||||
rowid,
|
||||
]);
|
||||
throw new Error("failed to find vm " + vmid);
|
||||
}
|
||||
}
|
||||
if (vm?.host) {
|
||||
// Save the VM data
|
||||
await postgres.query(
|
||||
`UPDATE vbrowser SET data = $1 WHERE id = $2`,
|
||||
[vm, rowid],
|
||||
);
|
||||
}
|
||||
}
|
||||
if (!vm?.host) {
|
||||
console.log(
|
||||
"[CHECKSTAGING] %s: no host for vm %s",
|
||||
this.getPoolName(),
|
||||
vmid,
|
||||
);
|
||||
throw new Error("no host for vm " + vmid);
|
||||
}
|
||||
if (retryCount % 150 === 0) {
|
||||
console.log(
|
||||
"[CHECKSTAGING] %s: %s poweron, attach to network",
|
||||
this.getPoolName(),
|
||||
vmid,
|
||||
);
|
||||
this.powerOn(vmid);
|
||||
//this.attachToNetwork(vmid);
|
||||
}
|
||||
if (retryCount % 180 === 0) {
|
||||
console.log("[CHECKSTAGING]", this.getPoolName(), "giving up:", vmid);
|
||||
redisCount("vBrowserStagingFails");
|
||||
await redis?.lpush("vBrowserStageFails", vmid);
|
||||
await redis?.ltrim("vBrowserStageFails", 0, 19);
|
||||
// VM didn't come up. set image to null so we reimage
|
||||
await postgres.query(
|
||||
`UPDATE vbrowser SET image = NULL WHERE pool = $1 AND vmid = $2`,
|
||||
[this.getPoolName(), vmid],
|
||||
);
|
||||
await this.resetVM(vmid);
|
||||
}
|
||||
if (retryCount >= 180) {
|
||||
throw new Error("too many attempts on vm " + vmid);
|
||||
}
|
||||
const ready = await this.checkVMReady(vm.host);
|
||||
if (
|
||||
ready ||
|
||||
retryCount % (config.NODE_ENV === "development" ? 1 : 30) === 0
|
||||
) {
|
||||
console.log(
|
||||
"[CHECKSTAGING] %s: [ready: %s] [vmid: %s] [retries: %s] [host: %s]",
|
||||
this.getPoolName(),
|
||||
ready,
|
||||
vmid,
|
||||
retryCount,
|
||||
vm?.host,
|
||||
);
|
||||
}
|
||||
if (ready) {
|
||||
const passUrl =
|
||||
"https://" +
|
||||
(vm.host.includes("/")
|
||||
? vm.host.replace("/", "/password")
|
||||
: vm.host + "/password") +
|
||||
(vm.host.includes("?") ? "&" : "?") +
|
||||
"key=" +
|
||||
config.VBROWSER_ADMIN_KEY;
|
||||
// console.log(passUrl);
|
||||
const resp2 = await axios({
|
||||
method: "GET",
|
||||
url: passUrl,
|
||||
timeout: 1000,
|
||||
});
|
||||
const pass = resp2.data;
|
||||
// password is a uuid so it should never match the previous value, if it does we probably didn't reset properly
|
||||
const { rows } = await postgres.query(
|
||||
`UPDATE vbrowser SET state = 'available', pass = $2 WHERE id = $1 AND pass IS DISTINCT FROM $2`,
|
||||
[rowid, pass],
|
||||
);
|
||||
// console.log(rows);
|
||||
await redis?.lpush("vBrowserStageRetries", retryCount);
|
||||
await redis?.ltrim("vBrowserStageRetries", 0, 19);
|
||||
}
|
||||
return [vmid, retryCount, ready].join(",");
|
||||
});
|
||||
// TODO log something if we timeout
|
||||
const result = await Promise.race([
|
||||
Promise.allSettled(stagingPromises),
|
||||
new Promise((resolve) => setTimeout(resolve, 30000)),
|
||||
]);
|
||||
return result;
|
||||
};
|
||||
|
||||
console.log("[VMWORKER] %s: starting background jobs", this.getPoolName());
|
||||
|
||||
setInterval(resizeVMGroupIncr, incrInterval);
|
||||
setInterval(resizeVMGroupDecr, decrInterval);
|
||||
setInterval(async () => {
|
||||
console.log(
|
||||
"[STATS] %s: currentSize %s, available %s, staging %s, target %s",
|
||||
this.getPoolName(),
|
||||
await this.getCurrentSize(),
|
||||
await this.getAvailableCount(),
|
||||
await this.getStagingCount(),
|
||||
this.getTargetBuffer(),
|
||||
);
|
||||
}, 10000);
|
||||
|
||||
// The following may take a while per iteration
|
||||
// Use while loop and delay between iterations rather than setInterval to avoid stacking requests
|
||||
setImmediate(async () => {
|
||||
while (true) {
|
||||
console.time(this.getPoolName() + ":cleanup");
|
||||
try {
|
||||
await cleanupVMGroup();
|
||||
} catch (e: any) {
|
||||
console.warn(
|
||||
"[CLEANUPVMGROUP-ERROR]",
|
||||
this.getPoolName(),
|
||||
e.response?.data,
|
||||
);
|
||||
}
|
||||
console.timeEnd(this.getPoolName() + ":cleanup");
|
||||
await new Promise((resolve) => setTimeout(resolve, cleanupInterval));
|
||||
}
|
||||
});
|
||||
|
||||
setImmediate(async () => {
|
||||
while (true) {
|
||||
// console.time(this.getPoolName() + ':checkstaging');
|
||||
try {
|
||||
await checkStaging();
|
||||
} catch (e) {
|
||||
console.warn("[CHECKSTAGING-ERROR]", this.getPoolName(), e);
|
||||
}
|
||||
// console.timeEnd(this.getPoolName() + ':checkstaging');
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
public abstract id: string;
|
||||
protected abstract size: string;
|
||||
protected abstract largeSize: string;
|
||||
protected abstract minRetries: number;
|
||||
protected abstract reuseVMs: boolean;
|
||||
protected abstract imageId: string;
|
||||
protected abstract startVM: (name: string) => Promise<string>;
|
||||
protected abstract rebootVM: (id: string) => Promise<void>;
|
||||
protected abstract reimageVM: (id: string) => Promise<void>;
|
||||
protected abstract terminateVM: (id: string) => Promise<void>;
|
||||
public abstract getVM: (id: string) => Promise<VM>;
|
||||
protected abstract listVMs: (filter: string) => Promise<VM[]>;
|
||||
protected abstract powerOn: (id: string) => Promise<void>;
|
||||
protected abstract attachToNetwork: (id: string) => Promise<void>;
|
||||
protected abstract mapServerObject: (server: any) => VM;
|
||||
public abstract updateSnapshot: () => Promise<string>;
|
||||
}
|
||||
|
||||
function pointInInterval24(x: number, a: number, b: number) {
|
||||
return nonNegativeMod(x - a, 24) <= nonNegativeMod(b - a, 24);
|
||||
}
|
||||
|
||||
function nonNegativeMod(n: number, m: number) {
|
||||
return ((n % m) + m) % m;
|
||||
}
|
||||
|
||||
export interface VM {
|
||||
id: string;
|
||||
host: string;
|
||||
provider: string;
|
||||
large: boolean;
|
||||
region: string;
|
||||
}
|
||||
|
||||
export interface AssignedVM extends VM {
|
||||
pass: string;
|
||||
assignTime: number;
|
||||
controllerClient?: string;
|
||||
creatorUID?: string;
|
||||
creatorClientID?: string;
|
||||
}
|
||||
|
||||
export interface VMManagers {
|
||||
standard: VMManager | null;
|
||||
large: VMManager | null;
|
||||
US: VMManager | null;
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
import config from "../config.ts";
|
||||
import axios from "axios";
|
||||
import { VMManager, type VM } from "./base.ts";
|
||||
|
||||
const DO_TOKEN = config.DO_TOKEN;
|
||||
const region = "sfo3";
|
||||
const gatewayHost = config.DO_GATEWAY;
|
||||
const sshKeys = config.DO_SSH_KEYS.split(",");
|
||||
|
||||
export class DigitalOcean extends VMManager {
|
||||
size = "s-2vcpu-2gb"; // s-1vcpu-1gb, s-1vcpu-2gb, s-2vcpu-2gb, s-2vcpu-4gb, c-2, s-4vcpu-8gb
|
||||
largeSize = "s-4vcpu-8gb";
|
||||
minRetries = 5;
|
||||
reuseVMs = true;
|
||||
id = "DO";
|
||||
imageId = config.DO_IMAGE;
|
||||
startVM = async (name: string) => {
|
||||
const response = await axios({
|
||||
method: "POST",
|
||||
url: `https://api.digitalocean.com/v2/droplets`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + DO_TOKEN,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
data: {
|
||||
name: name,
|
||||
region: region,
|
||||
size: this.isLarge ? this.largeSize : this.size,
|
||||
image: Number(this.imageId),
|
||||
ssh_keys: sshKeys,
|
||||
private_networking: true,
|
||||
// user_data: cloudInit(),
|
||||
tags: [this.getTag()],
|
||||
},
|
||||
});
|
||||
const id = response.data.droplet.id;
|
||||
return id;
|
||||
};
|
||||
|
||||
terminateVM = async (id: string) => {
|
||||
const response = await axios({
|
||||
method: "DELETE",
|
||||
url: `https://api.digitalocean.com/v2/droplets/${id}`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + DO_TOKEN,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
rebootVM = async (id: string) => {
|
||||
// Reboot the VM
|
||||
const response2 = await axios({
|
||||
method: "POST",
|
||||
url: `https://api.digitalocean.com/v2/droplets/${id}/actions`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + DO_TOKEN,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
data: {
|
||||
type: "reboot",
|
||||
},
|
||||
});
|
||||
return;
|
||||
};
|
||||
|
||||
reimageVM = async (id: string) => {
|
||||
// Rebuild the VM
|
||||
const response2 = await axios({
|
||||
method: "POST",
|
||||
url: `https://api.digitalocean.com/v2/droplets/${id}/actions`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + DO_TOKEN,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
data: {
|
||||
type: "rebuild",
|
||||
image: Number(this.imageId),
|
||||
},
|
||||
});
|
||||
return;
|
||||
};
|
||||
|
||||
getVM = async (id: string) => {
|
||||
const response = await axios({
|
||||
method: "GET",
|
||||
url: `https://api.digitalocean.com/v2/droplets/${id}`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + DO_TOKEN,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
let server = this.mapServerObject(response.data.droplet);
|
||||
return server;
|
||||
};
|
||||
|
||||
listVMs = async (filter: string) => {
|
||||
// console.log(filter, tags);
|
||||
const response = await axios({
|
||||
method: "GET",
|
||||
url: `https://api.digitalocean.com/v2/droplets`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + DO_TOKEN,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
params: {
|
||||
// TODO need to update if over 100 results
|
||||
per_page: 100,
|
||||
tag_name: filter,
|
||||
},
|
||||
});
|
||||
return response.data.droplets.map(this.mapServerObject);
|
||||
};
|
||||
|
||||
powerOn = async (_id: string) => {};
|
||||
|
||||
attachToNetwork = async (_id: string) => {};
|
||||
|
||||
updateSnapshot = async () => {
|
||||
return "";
|
||||
};
|
||||
|
||||
mapServerObject = (server: any): VM => {
|
||||
// const ip = server.networks.v4.find(
|
||||
// (network: any) => network.type === 'private',
|
||||
// )?.ip_address;
|
||||
const ip = server.networks.v4.find(
|
||||
(network: any) => network.type === "public",
|
||||
)?.ip_address;
|
||||
return {
|
||||
id: server.id?.toString(),
|
||||
// The gateway handles SSL termination and proxies to the private IP
|
||||
host: ip ? `${gatewayHost}/?ip=${ip}` : "",
|
||||
provider: this.id,
|
||||
large: this.isLarge,
|
||||
region: this.region,
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,134 @@
|
||||
// This assumes an installation of Docker exists on the given host
|
||||
// and that host is configured to accept our SSH key
|
||||
import config from "../config.ts";
|
||||
import { VMManager, type VM } from "./base.ts";
|
||||
import { imageName } from "./utils.ts";
|
||||
import fs from "node:fs";
|
||||
import { homedir } from "node:os";
|
||||
import { NodeSSH } from "node-ssh";
|
||||
|
||||
export class Docker extends VMManager {
|
||||
// TODO support multiple Docker providers in the pool config with same region
|
||||
size = "";
|
||||
largeSize = "";
|
||||
minRetries = 0;
|
||||
reuseVMs = false;
|
||||
id = "Docker";
|
||||
ssh: NodeSSH | undefined = undefined;
|
||||
imageId = imageName;
|
||||
|
||||
getSSH = async () => {
|
||||
if (this.ssh && this.ssh.isConnected()) {
|
||||
return this.ssh;
|
||||
}
|
||||
const sshConfig = {
|
||||
username: config.DOCKER_VM_HOST_SSH_USER,
|
||||
host: this.hostname,
|
||||
// The private key the Docker host is configured to accept
|
||||
privateKey: config.DOCKER_VM_HOST_SSH_KEY_BASE64
|
||||
? Buffer.from(config.DOCKER_VM_HOST_SSH_KEY_BASE64, "base64").toString()
|
||||
: fs.readFileSync(homedir() + "/.ssh/id_rsa").toString(),
|
||||
};
|
||||
this.ssh = new NodeSSH();
|
||||
await this.ssh.connect(sshConfig);
|
||||
return this.ssh;
|
||||
};
|
||||
|
||||
startVM = async (name: string) => {
|
||||
const tag = this.getTag();
|
||||
const conn = await this.getSSH();
|
||||
// If in development, have neko share the same SSL cert as the other services
|
||||
// If in production, they are probably on different hosts and neko is behind a reverse proxy for SSL termination
|
||||
const sslEnv =
|
||||
config.NODE_ENV === "development" &&
|
||||
config.SSL_KEY_FILE &&
|
||||
config.SSL_CRT_FILE
|
||||
? `-e NEKO_KEY="${config.SSL_KEY_FILE}" -e NEKO_CERT="${config.SSL_CRT_FILE}"`
|
||||
: "";
|
||||
const { stdout, stderr } = await conn.execCommand(
|
||||
`
|
||||
#!/bin/bash
|
||||
set -e
|
||||
PORT=$(comm -23 <(seq 5000 5063 | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2 | sort -u) | sort -n | head -n 1)
|
||||
INDEX=$(($PORT - 5000))
|
||||
UDP_START=$((59000+$INDEX*100))
|
||||
UDP_END=$((59099+$INDEX*100))
|
||||
docker run -d --rm --name=${name} --memory="2g" --cpus="2" -p $PORT:$PORT -p $UDP_START-$UDP_END:$UDP_START-$UDP_END/udp -v /etc/letsencrypt:/etc/letsencrypt -l ${tag} -l index=$INDEX --log-opt max-size=1g --shm-size=1g --cap-add="SYS_ADMIN" ${sslEnv} -e DISPLAY=":99.0" -e NEKO_PASSWORD=${name} -e NEKO_PASSWORD_ADMIN=${name} -e NEKO_ADMIN_KEY=${config.VBROWSER_ADMIN_KEY} -e NEKO_BIND=":$PORT" -e NEKO_EPR=":$UDP_START-$UDP_END" -e NEKO_H264="1" ${imageName}
|
||||
`,
|
||||
);
|
||||
console.log(stdout, stderr);
|
||||
return stdout.trim();
|
||||
};
|
||||
|
||||
terminateVM = async (id: string) => {
|
||||
const conn = await this.getSSH();
|
||||
const { stdout, stderr } = await conn.execCommand(`docker rm -fv ${id}`);
|
||||
console.log(stdout, stderr);
|
||||
return;
|
||||
};
|
||||
|
||||
rebootVM = async (id: string) => {
|
||||
// Docker containers aren't set to reuse, so do nothing (reset will terminate)
|
||||
};
|
||||
|
||||
reimageVM = async (id: string) => {
|
||||
const conn = await this.getSSH();
|
||||
const { stdout, stderr } = await conn.execCommand(
|
||||
`docker pull ${this.imageId}`,
|
||||
);
|
||||
console.log(stdout, stderr);
|
||||
// The container is out of date. Delete it
|
||||
this.terminateVMWrapper(id);
|
||||
return;
|
||||
};
|
||||
|
||||
getVM = async (id: string) => {
|
||||
const conn = await this.getSSH();
|
||||
const { stdout } = await conn.execCommand(`docker inspect ${id}`);
|
||||
let data = null;
|
||||
try {
|
||||
data = JSON.parse(stdout)[0];
|
||||
if (!data) {
|
||||
throw new Error("no container with this ID found");
|
||||
}
|
||||
} catch {
|
||||
console.warn(stdout);
|
||||
throw new Error("failed to parse json");
|
||||
}
|
||||
let server = this.mapServerObject(data);
|
||||
return server;
|
||||
};
|
||||
|
||||
listVMs = async (filter: string) => {
|
||||
const conn = await this.getSSH();
|
||||
const listCmd = `docker inspect $(docker ps --filter label=${filter} --quiet --no-trunc)`;
|
||||
const { stdout } = await conn.execCommand(listCmd);
|
||||
if (!stdout) {
|
||||
return [];
|
||||
}
|
||||
let data = [];
|
||||
try {
|
||||
data = JSON.parse(stdout);
|
||||
} catch (e) {
|
||||
console.warn(stdout);
|
||||
throw new Error("failed to parse json");
|
||||
}
|
||||
return data.map(this.mapServerObject);
|
||||
};
|
||||
|
||||
powerOn = async (id: string) => {};
|
||||
|
||||
attachToNetwork = async (id: string) => {};
|
||||
|
||||
updateSnapshot = async () => {
|
||||
return "";
|
||||
};
|
||||
|
||||
mapServerObject = (server: any): VM => ({
|
||||
id: server.Id,
|
||||
host: `${this.hostname}:${5000 + Number(server.Config?.Labels?.index)}`,
|
||||
provider: this.id,
|
||||
large: this.isLarge,
|
||||
region: this.region,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
import config from "../config.ts";
|
||||
import axios from "axios";
|
||||
import { VMManager, type VM } from "./base.ts";
|
||||
import fs from "node:fs";
|
||||
import { redis } from "../utils/redis.ts";
|
||||
|
||||
const HETZNER_TOKEN = config.HETZNER_TOKEN;
|
||||
const sshKeys = config.HETZNER_SSH_KEYS.split(",").map(Number);
|
||||
|
||||
export class Hetzner extends VMManager {
|
||||
size = "cpx11"; // cpx11, cx22 (not available in US yet)
|
||||
largeSize = "cpx31"; // cpx21, cpx31, ccx13, ccx23, cx32
|
||||
minRetries = 5;
|
||||
reuseVMs = true;
|
||||
id = "Hetzner";
|
||||
gateway = config.HETZNER_GATEWAY;
|
||||
imageId = config.HETZNER_IMAGE;
|
||||
|
||||
private getRandomDatacenter() {
|
||||
// US
|
||||
let datacenters = ["ash"];
|
||||
if (this.region === "USW") {
|
||||
datacenters = ["hil"];
|
||||
} else if (this.region === "EU") {
|
||||
datacenters = ["nbg1", "fsn1", "hel1"];
|
||||
}
|
||||
return datacenters[Math.floor(Math.random() * datacenters.length)];
|
||||
}
|
||||
|
||||
startVM = async (name: string) => {
|
||||
const data = {
|
||||
name: name,
|
||||
server_type: this.isLarge ? this.largeSize : this.size,
|
||||
start_after_create: true,
|
||||
image: Number(this.imageId),
|
||||
ssh_keys: sshKeys,
|
||||
public_net: {
|
||||
enable_ipv4: true,
|
||||
enable_ipv6: false,
|
||||
},
|
||||
// networks: [
|
||||
// this.networks[Math.floor(Math.random() * this.networks.length)],
|
||||
// ],
|
||||
// user_data: `replace with vbrowser.sh startup script if we want to boot vbrowser on instance creation (won't trigger on rebuild/restart)`
|
||||
labels: {
|
||||
[this.getTag()]: "1",
|
||||
},
|
||||
location: this.getRandomDatacenter(),
|
||||
};
|
||||
const response = await axios({
|
||||
method: "POST",
|
||||
url: `https://api.hetzner.cloud/v1/servers`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + HETZNER_TOKEN,
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "test",
|
||||
},
|
||||
data,
|
||||
});
|
||||
const id = response.data.server.id;
|
||||
return id;
|
||||
};
|
||||
|
||||
terminateVM = async (id: string) => {
|
||||
await axios({
|
||||
method: "DELETE",
|
||||
url: `https://api.hetzner.cloud/v1/servers/${id}`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + HETZNER_TOKEN,
|
||||
"User-Agent": "test",
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
rebootVM = async (id: string) => {
|
||||
// Reboot the VM
|
||||
await axios({
|
||||
method: "POST",
|
||||
url: `https://api.hetzner.cloud/v1/servers/${id}/actions/reboot`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + HETZNER_TOKEN,
|
||||
"User-Agent": "test",
|
||||
},
|
||||
});
|
||||
return;
|
||||
};
|
||||
|
||||
reimageVM = async (id: string) => {
|
||||
// Rebuild the VM
|
||||
await axios({
|
||||
method: "POST",
|
||||
url: `https://api.hetzner.cloud/v1/servers/${id}/actions/rebuild`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + HETZNER_TOKEN,
|
||||
"User-Agent": "test",
|
||||
},
|
||||
data: {
|
||||
image: Number(this.imageId),
|
||||
},
|
||||
});
|
||||
return;
|
||||
};
|
||||
|
||||
getVM = async (id: string) => {
|
||||
const response: any = await axios({
|
||||
method: "GET",
|
||||
url: `https://api.hetzner.cloud/v1/servers/${id}`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + HETZNER_TOKEN,
|
||||
"User-Agent": "test",
|
||||
},
|
||||
});
|
||||
console.log(
|
||||
"[GETVM] %s: %s rate limit remaining",
|
||||
id,
|
||||
response?.headers["ratelimit-remaining"],
|
||||
);
|
||||
await redis?.set(
|
||||
"hetznerApiRemaining",
|
||||
response?.headers["ratelimit-remaining"],
|
||||
);
|
||||
const server = this.mapServerObject(response.data.server);
|
||||
return server;
|
||||
};
|
||||
|
||||
listVMs = async (filter: string) => {
|
||||
const limit = this.getLimitSize();
|
||||
const pageCount = Math.ceil((limit || 1) / 50);
|
||||
const pages = Array.from(Array(pageCount).keys()).map((i) => i + 1);
|
||||
const responses: any[] = await Promise.all(
|
||||
pages.map((page) =>
|
||||
axios({
|
||||
method: "GET",
|
||||
url: `https://api.hetzner.cloud/v1/servers`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + HETZNER_TOKEN,
|
||||
"User-Agent": "test",
|
||||
},
|
||||
params: {
|
||||
sort: "id:asc",
|
||||
page,
|
||||
per_page: 50,
|
||||
label_selector: filter,
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
const responsesMapped = responses.map((response) =>
|
||||
response.data.servers.map(this.mapServerObject),
|
||||
);
|
||||
return responsesMapped.flat();
|
||||
};
|
||||
|
||||
powerOn = async (id: string) => {
|
||||
// Poweron the server (usually not needed)
|
||||
try {
|
||||
await axios({
|
||||
method: "POST",
|
||||
url: `https://api.hetzner.cloud/v1/servers/${id}/actions/poweron`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + HETZNER_TOKEN,
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "test",
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
console.log("%s failed to poweron", id);
|
||||
}
|
||||
};
|
||||
|
||||
attachToNetwork = async (id: string) => {
|
||||
// // Attach server to network (usually not needed)
|
||||
// try {
|
||||
// const response: any = await axios({
|
||||
// method: 'GET',
|
||||
// url: `https://api.hetzner.cloud/v1/servers/${id}`,
|
||||
// headers: {
|
||||
// Authorization: 'Bearer ' + HETZNER_TOKEN,
|
||||
// },
|
||||
// });
|
||||
// if (response.data.server.private_net?.[0] == null) {
|
||||
// await axios({
|
||||
// method: 'POST',
|
||||
// url: `https://api.hetzner.cloud/v1/servers/${id}/actions/attach_to_network`,
|
||||
// headers: {
|
||||
// Authorization: 'Bearer ' + HETZNER_TOKEN,
|
||||
// 'Content-Type': 'application/json',
|
||||
// },
|
||||
// data: {
|
||||
// network:
|
||||
// this.networks[Math.floor(Math.random() * this.networks.length)],
|
||||
// },
|
||||
// });
|
||||
// }
|
||||
// } catch (e: any) {
|
||||
// console.log('%s failed to attach to network', id);
|
||||
// console.log(e.response?.data);
|
||||
// }
|
||||
};
|
||||
|
||||
updateSnapshot = async () => {
|
||||
const response = await axios({
|
||||
method: "POST",
|
||||
url: `https://api.hetzner.cloud/v1/servers`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + HETZNER_TOKEN,
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "test",
|
||||
},
|
||||
data: {
|
||||
name: "vBrowserSnapshot",
|
||||
server_type: "cpx11",
|
||||
start_after_create: true,
|
||||
image: "docker-ce", // 15512617 for Ubuntu 20.04
|
||||
ssh_keys: sshKeys,
|
||||
user_data: fs
|
||||
.readFileSync(import.meta.dirname + "/../../dev/vbrowser.sh")
|
||||
.toString()
|
||||
.replace("{VBROWSER_ADMIN_KEY}", config.VBROWSER_ADMIN_KEY),
|
||||
location: this.getRandomDatacenter(),
|
||||
},
|
||||
});
|
||||
const id = response.data.server.id;
|
||||
await new Promise((resolve) => setTimeout(resolve, 4 * 60 * 1000));
|
||||
// Validate snapshot server was created successfully
|
||||
// const response3 = await axios(
|
||||
// 'http://' + response.data.server.public_net?.ipv4?.ip + ':5000'
|
||||
// );
|
||||
const response2 = await axios({
|
||||
method: "POST",
|
||||
url: `https://api.hetzner.cloud/v1/servers/${id}/actions/create_image`,
|
||||
headers: {
|
||||
Authorization: "Bearer " + HETZNER_TOKEN,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
const imageId = response2.data.image.id;
|
||||
await this.terminateVM(id);
|
||||
return imageId;
|
||||
};
|
||||
|
||||
mapServerObject = (server: any): VM => {
|
||||
const public_ip = server.public_net?.ipv4?.ip;
|
||||
// const private_ip = server.private_net?.[0]?.ip;
|
||||
const ip = public_ip;
|
||||
// We can use either the public or private IP for communicating between gateway and VM
|
||||
// Only signaling traffic goes through here since the video is transmitted over WebRTC
|
||||
// The private IP requires the server and gateway to be on the same network and there is a limit to the number of servers allowed
|
||||
return {
|
||||
id: server.id?.toString(),
|
||||
// The gateway handles SSL termination and proxies to the private IP
|
||||
host: ip ? `${this.gateway}/?ip=${ip}` : "",
|
||||
provider: this.id,
|
||||
large: this.isLarge,
|
||||
region: this.region,
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
import config from "../config.ts";
|
||||
import axios from "axios";
|
||||
import { VMManager, type VM } from "./base.ts";
|
||||
|
||||
const SCW_SECRET_KEY = config.SCW_SECRET_KEY;
|
||||
const SCW_ORGANIZATION_ID = config.SCW_ORGANIZATION_ID;
|
||||
const region = "nl-ams-1"; //fr-par-1
|
||||
const gatewayHost = config.SCW_GATEWAY;
|
||||
|
||||
export class Scaleway extends VMManager {
|
||||
size = "DEV1-S"; // DEV1-S, DEV1-M, DEV1-L, GP1-XS
|
||||
largeSize = "DEV1-M";
|
||||
minRetries = 5;
|
||||
reuseVMs = true;
|
||||
id = "Scaleway";
|
||||
imageId = config.SCW_IMAGE;
|
||||
startVM = async (name: string) => {
|
||||
const response = await axios({
|
||||
method: "POST",
|
||||
url: `https://api.scaleway.com/instance/v1/zones/${region}/servers`,
|
||||
headers: {
|
||||
"X-Auth-Token": SCW_SECRET_KEY,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
data: {
|
||||
name: name,
|
||||
dynamic_ip_required: true,
|
||||
commercial_type: this.isLarge ? this.largeSize : this.size,
|
||||
image: this.imageId,
|
||||
volumes: {},
|
||||
organization: SCW_ORGANIZATION_ID,
|
||||
tags: [this.getTag()],
|
||||
},
|
||||
});
|
||||
// console.log(response.data);
|
||||
const id = response.data.server.id;
|
||||
const response2 = await axios({
|
||||
method: "PATCH",
|
||||
url: `https://api.scaleway.com/instance/v1/zones/${region}/servers/${id}/user_data/cloud-init`,
|
||||
headers: {
|
||||
"X-Auth-Token": SCW_SECRET_KEY,
|
||||
"Content-Type": "text/plain",
|
||||
},
|
||||
// set userdata for boot action
|
||||
//data: cloudInit(),
|
||||
});
|
||||
// console.log(response2.data);
|
||||
// boot the instance
|
||||
const response3 = await axios({
|
||||
method: "POST",
|
||||
url: `https://api.scaleway.com/instance/v1/zones/${region}/servers/${id}/action`,
|
||||
headers: {
|
||||
"X-Auth-Token": SCW_SECRET_KEY,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
data: {
|
||||
action: "poweron",
|
||||
},
|
||||
});
|
||||
// console.log(response3.data);
|
||||
return id;
|
||||
};
|
||||
|
||||
terminateVM = async (id: string) => {
|
||||
const response = await axios({
|
||||
method: "POST",
|
||||
url: `https://api.scaleway.com/instance/v1/zones/${region}/servers/${id}/action`,
|
||||
headers: {
|
||||
"X-Auth-Token": SCW_SECRET_KEY,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
data: {
|
||||
action: "terminate",
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
rebootVM = async (id: string) => {
|
||||
// Reboot the VM (also destroys the Docker container since it has --rm flag)
|
||||
const response2 = await axios({
|
||||
method: "POST",
|
||||
url: `https://api.scaleway.com/instance/v1/zones/${region}/servers/${id}/action`,
|
||||
headers: {
|
||||
"X-Auth-Token": SCW_SECRET_KEY,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
data: {
|
||||
action: "reboot",
|
||||
},
|
||||
});
|
||||
return;
|
||||
};
|
||||
|
||||
reimageVM = async (id: string) => {
|
||||
// Scaleway doesn't have a reimage/rebuild command. Delete the VM
|
||||
await this.terminateVMWrapper(id);
|
||||
};
|
||||
|
||||
getVM = async (id: string) => {
|
||||
const response = await axios({
|
||||
method: "GET",
|
||||
url: `https://api.scaleway.com/instance/v1/zones/${region}/servers/${id}`,
|
||||
headers: {
|
||||
"X-Auth-Token": SCW_SECRET_KEY,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
let server = this.mapServerObject(response.data.server);
|
||||
return server;
|
||||
};
|
||||
|
||||
listVMs = async (filter: string) => {
|
||||
const limit = this.getLimitSize();
|
||||
const pageCount = Math.ceil((limit || 1) / 100);
|
||||
const pages = Array.from(Array(pageCount).keys()).map((i) => i + 1);
|
||||
const responses: any[] = await Promise.all(
|
||||
pages.map((page) =>
|
||||
axios({
|
||||
method: "GET",
|
||||
url: `https://api.scaleway.com/instance/v1/zones/${region}/servers`,
|
||||
headers: {
|
||||
"X-Auth-Token": SCW_SECRET_KEY,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
params: {
|
||||
page,
|
||||
per_page: 100,
|
||||
tags: filter,
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
const responsesMapped = responses.map((response) =>
|
||||
response.data.servers.map(this.mapServerObject),
|
||||
);
|
||||
return responsesMapped.flat();
|
||||
};
|
||||
|
||||
powerOn = async (_id: string) => {};
|
||||
|
||||
attachToNetwork = async (_id: string) => {};
|
||||
|
||||
updateSnapshot = async () => {
|
||||
return "";
|
||||
};
|
||||
|
||||
mapServerObject = (server: any): VM => {
|
||||
// const ip = server.private_ip;
|
||||
const ip = server.public_ip?.address;
|
||||
return {
|
||||
id: server.id,
|
||||
// The gateway handles SSL termination and proxies to the private IP
|
||||
host: ip ? `${gatewayHost}/?ip=${ip}` : "",
|
||||
provider: this.id,
|
||||
large: this.isLarge,
|
||||
region: this.region,
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { type AssignedVM, VMManager } from "./base.ts";
|
||||
import config from "../config.ts";
|
||||
import { Scaleway } from "./scaleway.ts";
|
||||
import { Hetzner } from "./hetzner.ts";
|
||||
import { DigitalOcean } from "./digitalocean.ts";
|
||||
import { Docker } from "./docker.ts";
|
||||
|
||||
// Chromium on ARM: ghcr.io/howardchung/vbrowser/arm-chromium
|
||||
export const imageName = "howardc93/vbrowser";
|
||||
|
||||
export type PoolRegion = "US" | "USW" | "EU";
|
||||
export type PoolConfig = {
|
||||
provider: string;
|
||||
isLarge: boolean;
|
||||
region: PoolRegion;
|
||||
limitSize: number | undefined;
|
||||
minSize: number | undefined;
|
||||
hostname: string | undefined;
|
||||
};
|
||||
|
||||
function createVMManager(poolConfig: PoolConfig): VMManager {
|
||||
let vmManager: VMManager | null = null;
|
||||
if (
|
||||
config.SCW_SECRET_KEY &&
|
||||
config.SCW_ORGANIZATION_ID &&
|
||||
config.SCW_IMAGE &&
|
||||
config.SCW_GATEWAY &&
|
||||
poolConfig.provider === "Scaleway"
|
||||
) {
|
||||
vmManager = new Scaleway(poolConfig);
|
||||
} else if (
|
||||
config.HETZNER_TOKEN &&
|
||||
config.HETZNER_IMAGE &&
|
||||
config.HETZNER_GATEWAY &&
|
||||
poolConfig.provider === "Hetzner"
|
||||
) {
|
||||
vmManager = new Hetzner(poolConfig);
|
||||
} else if (
|
||||
config.DO_TOKEN &&
|
||||
config.DO_IMAGE &&
|
||||
config.DO_GATEWAY &&
|
||||
poolConfig.provider === "DO"
|
||||
) {
|
||||
vmManager = new DigitalOcean(poolConfig);
|
||||
} else if (poolConfig.provider === "Docker") {
|
||||
vmManager = new Docker(poolConfig);
|
||||
}
|
||||
if (!vmManager) {
|
||||
throw new Error("failed to create vmManager");
|
||||
}
|
||||
return vmManager;
|
||||
}
|
||||
|
||||
export function getVMManagerConfig(): PoolConfig[] {
|
||||
return config.VM_MANAGER_CONFIG.split(",")
|
||||
.filter(Boolean)
|
||||
.map((c) => {
|
||||
const split = c.split(":");
|
||||
return {
|
||||
provider: split[0],
|
||||
isLarge: split[1] === "large",
|
||||
region: split[2] as PoolRegion,
|
||||
minSize: Number(split[3]),
|
||||
limitSize: Number(split[4]),
|
||||
hostname: split[5],
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function getBgVMManagers(): { [key: string]: VMManager } {
|
||||
const result: { [key: string]: VMManager } = {};
|
||||
const conf = getVMManagerConfig();
|
||||
conf.forEach((c) => {
|
||||
const mgr = createVMManager(c);
|
||||
if (mgr) {
|
||||
result[mgr.getPoolName()] = mgr;
|
||||
}
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
export function getSessionLimitSeconds(isLarge: boolean) {
|
||||
return isLarge
|
||||
? config.VBROWSER_SESSION_SECONDS_LARGE
|
||||
: config.VBROWSER_SESSION_SECONDS;
|
||||
}
|
||||
Reference in New Issue
Block a user