This commit is contained in:
2026-06-21 03:53:06 +03:00
commit f1dd557c5d
147 changed files with 34363 additions and 0 deletions
+73
View File
@@ -0,0 +1,73 @@
import config from "../config.ts";
import admin from "firebase-admin";
if (config.FIREBASE_ADMIN_SDK_CONFIG) {
admin.initializeApp({
credential: admin.credential.cert(
JSON.parse(config.FIREBASE_ADMIN_SDK_CONFIG),
),
databaseURL: config.FIREBASE_DATABASE_URL,
});
}
export async function validateUserToken(uid: string, token: string) {
if (!config.FIREBASE_ADMIN_SDK_CONFIG) {
return undefined;
}
if (!token) {
return undefined;
}
try {
const decoded = await admin.auth().verifyIdToken(token);
if (uid !== decoded.uid) {
// Valid but for wrong user
return undefined;
}
return decoded;
} catch (e) {
// Promise rejects if verification failed
console.log(e);
return undefined;
}
}
export async function writeData(key: string, value: string) {
if (!config.FIREBASE_ADMIN_SDK_CONFIG) {
return;
}
await admin.database().ref(key).set(value);
}
export async function getUserByEmail(email: string) {
if (!config.FIREBASE_ADMIN_SDK_CONFIG) {
return null;
}
try {
return await admin.auth().getUserByEmail(email);
} catch (e: any) {
console.log(email, e.message);
}
return null;
}
export async function getUser(uid: string) {
if (!config.FIREBASE_ADMIN_SDK_CONFIG) {
return null;
}
return await admin.auth().getUser(uid);
}
export async function getUserEmail(uid: string) {
if (!config.FIREBASE_ADMIN_SDK_CONFIG) {
return null;
}
const user = await admin.auth().getUser(uid);
return user.email;
}
export async function deleteUser(uid: string) {
if (!config.FIREBASE_ADMIN_SDK_CONFIG) {
return null;
}
return admin.auth().deleteUser(uid);
}
+312
View File
@@ -0,0 +1,312 @@
import type { AssignedVM } from "../vm/base.ts";
import { postgres } from "./postgres.ts";
import os from "node:os";
import { getRedisCountDay, getRedisCountDayDistinct, redis } from "./redis.ts";
import config from "../config.ts";
import { apps } from "../ecosystem.config.js";
export async function getStats() {
const now = Date.now();
let currentUsers = 0;
// Render each shard metrics as its own object
const shardMetrics: Record<string, ShardMetric> = {};
const shardKeys = new Set(
apps.map((app) => `shardMetrics:${app.env?.SHARD ?? 0}`),
);
for (let key of shardKeys) {
const resp2 = await redis?.get(key);
if (resp2) {
shardMetrics[key] = JSON.parse(resp2);
currentUsers += shardMetrics[key].users;
}
}
// Count these from postgres data
let currentHttp = 0;
let currentVBrowser = 0;
let currentVBrowserLarge = 0;
let currentScreenShare = 0;
let currentFileShare = 0;
const currentRoomSizes: Record<string, number> = {};
const result = await postgres?.query<{
roomId: string;
creationTime: Date;
lastUpdateTime: Date;
vanity: string;
isSubRoom: boolean;
roomTitle: string;
roomDescription: string;
mediaPath: string;
owner: string;
password: string;
video: string;
videoTS: number;
vBrowser: AssignedVM;
creator: string;
lock: string;
}>(
`SELECT "roomId", "creationTime", "lastUpdateTime", vanity, "isSubRoom", "roomTitle", "roomDescription", "mediaPath", owner, password,
data->'video' as video, data->'videoTS' as "videoTS", data->'vBrowser' as "vBrowser", data->'creator' as creator, data->'lock' as lock
FROM room
WHERE "lastUpdateTime" > NOW() - INTERVAL '7 day'
AND length(data->>'video') > 0
ORDER BY "creationTime" DESC`,
);
const currentRoomData = await Promise.all(
(result?.rows ?? []).map(async (dbRoom) => {
const vBrowser = dbRoom.vBrowser;
if (vBrowser) {
currentVBrowser += 1;
}
if (vBrowser?.large) {
currentVBrowserLarge += 1;
}
const rosterLength = Number(
await redis?.get(`roomCounts:${dbRoom.roomId}`),
);
let roster = [];
if (rosterLength) {
currentRoomSizes[rosterLength] =
(currentRoomSizes[rosterLength] ?? 0) + 1;
const resp = await redis?.get(`roomRosters:${dbRoom.roomId}`);
if (resp) {
roster = JSON.parse(resp);
}
}
const obj = {
roomId: dbRoom.roomId,
video: dbRoom.video || undefined,
videoTS: dbRoom.videoTS || undefined,
creationTime: dbRoom.creationTime || undefined,
lastUpdateTime: dbRoom.lastUpdateTime || undefined,
vanity: dbRoom.vanity || undefined,
isSubRoom: dbRoom.isSubRoom || undefined,
owner: dbRoom.owner || undefined,
password: dbRoom.password || undefined,
roomTitle: dbRoom.roomTitle || undefined,
roomDescription: dbRoom.roomDescription || undefined,
mediaPath: dbRoom.mediaPath || undefined,
vBrowser,
vBrowserElapsed: vBrowser?.assignTime && now - vBrowser?.assignTime,
lock: dbRoom.lock || undefined,
creator: dbRoom.creator || undefined,
rosterLength,
roster,
};
if (obj.video?.startsWith("http") && rosterLength) {
currentHttp += 1;
}
if (obj.video?.startsWith("screenshare://") && rosterLength) {
currentScreenShare += 1;
}
if (obj.video?.startsWith("fileshare://") && rosterLength) {
currentFileShare += 1;
}
return obj;
}),
);
// Singleton stats below (same for all shards)
const currentVideoChat = 0;
const cpuUsage = os.loadavg()[1] * 100;
const redisUsage = Number(
(await redis?.info())
?.split("\n")
.find((line) => line.startsWith("used_memory:"))
?.split(":")[1]
.trim(),
);
const postgresUsage = Number(
(await postgres?.query(`SELECT pg_database_size('postgres');`))?.rows[0]
.pg_database_size,
);
const numPermaRooms = Number(
(await postgres?.query("SELECT count(1) from room WHERE owner IS NOT NULL"))
?.rows[0].count,
);
const numAllRooms = Number(
(await postgres?.query("SELECT count(1) from room"))?.rows[0].count,
);
const numSubs = Number(
(await postgres?.query("SELECT count(1) from subscriber"))?.rows[0].count,
);
const discordBotWatch = await getRedisCountDay("discordBotWatch");
const createRoomErrors = await getRedisCountDay("createRoomError");
const deleteAccounts = await getRedisCountDay("deleteAccount");
const chatMessages = await getRedisCountDay("chatMessages");
const addReactions = await getRedisCountDay("addReaction");
const hetznerApiRemaining = Number(await redis?.get("hetznerApiRemaining"));
const vBrowserStarts = await getRedisCountDay("vBrowserStarts");
const vBrowserLaunches = await getRedisCountDay("vBrowserLaunches");
const vBrowserFails = await getRedisCountDay("vBrowserFails");
const vBrowserStagingFails = await getRedisCountDay("vBrowserStagingFails");
const vBrowserReimages = await getRedisCountDay("vBrowserReimage");
const vBrowserCleanups = await getRedisCountDay("vBrowserCleanup");
const vBrowserStopTimeout = await getRedisCountDay(
"vBrowserTerminateTimeout",
);
const vBrowserStopEmpty = await getRedisCountDay("vBrowserTerminateEmpty");
const vBrowserStopManual = await getRedisCountDay("vBrowserTerminateManual");
const vBrowserStartMS = await redis?.lrange("vBrowserStartMS", 0, -1);
const vBrowserStageRetries = await redis?.lrange(
"vBrowserStageRetries",
0,
-1,
);
const vBrowserStageFails = await redis?.lrange("vBrowserStageFails", 0, -1);
const vBrowserSessionMS = await redis?.lrange("vBrowserSessionMS", 0, -1);
// const vBrowserVMLifetime = await redis?.lrange('vBrowserVMLifetime', 0, -1);
const proxyReqs = await getRedisCountDay("proxyReqs");
const urlStarts = await getRedisCountDay("urlStarts");
const streamStarts = await getRedisCountDay("streamStarts");
const convertStarts = await getRedisCountDay("convertStarts");
const playlistAdds = await getRedisCountDay("playlistAdds");
const screenShareStarts = await getRedisCountDay("screenShareStarts");
const fileShareStarts = await getRedisCountDay("fileShareStarts");
const mediasoupStarts = await getRedisCountDay("mediasoupStarts");
const videoChatStarts = await getRedisCountDay("videoChatStarts");
const connectStarts = await getRedisCountDay("connectStarts");
const connectStartsDistinct = await getRedisCountDayDistinct(
"connectStartsDistinct",
);
const subUploads = await getRedisCountDay("subUploads");
const subDownloadsOS = await getRedisCountDay("subDownloadsOS");
const subSearchesOS = await getRedisCountDay("subSearchesOS");
const youtubeSearch = await getRedisCountDay("youtubeSearch");
const vBrowserClientIDsCard = await redis?.zcard("vBrowserClientIDs");
const vBrowserUIDsCard = await redis?.zcard("vBrowserUIDs");
const createRoomPreloads = await getRedisCountDay("createRoomPreload");
const vBrowserClientIDs = altArrayToObject(
await redis?.zrevrangebyscore(
"vBrowserClientIDs",
"+inf",
"0",
"WITHSCORES",
"LIMIT",
0,
20,
),
);
const vBrowserUIDs = altArrayToObject(
await redis?.zrevrangebyscore(
"vBrowserUIDs",
"+inf",
"0",
"WITHSCORES",
"LIMIT",
0,
20,
),
);
const vBrowserClientIDMinutes = altArrayToObject(
await redis?.zrevrangebyscore(
"vBrowserClientIDMinutes",
"+inf",
"0",
"WITHSCORES",
"LIMIT",
0,
20,
),
);
const vBrowserUIDMinutes = altArrayToObject(
await redis?.zrevrangebyscore(
"vBrowserUIDMinutes",
"+inf",
"0",
"WITHSCORES",
"LIMIT",
0,
20,
),
);
// Fetch VM stats from vmworker
const resp = await fetch(
"http://localhost:" + config.VMWORKER_PORT + "/stats",
);
const vmManagerStats = await resp.json();
return {
...shardMetrics,
currentRoomSizes,
counts: {
currentUsers,
currentVideoChat,
currentVBrowser,
currentVBrowserLarge,
currentHttp,
currentScreenShare,
currentFileShare,
cpuUsage,
redisUsage,
postgresUsage,
numPermaRooms,
numAllRooms,
numSubs,
discordBotWatch,
createRoomErrors,
createRoomPreloads,
deleteAccounts,
chatMessages,
addReactions,
proxyReqs,
urlStarts,
streamStarts,
convertStarts,
playlistAdds,
screenShareStarts,
fileShareStarts,
mediasoupStarts,
subUploads,
subDownloadsOS,
subSearchesOS,
youtubeSearch,
videoChatStarts,
connectStarts,
connectStartsDistinct,
hetznerApiRemaining,
vBrowserStarts,
vBrowserLaunches,
vBrowserFails,
vBrowserStagingFails,
vBrowserReimages,
vBrowserCleanups,
vBrowserStopManual,
vBrowserStopEmpty,
vBrowserStopTimeout,
vBrowserClientIDsCard,
vBrowserUIDsCard,
},
// Stats object from vmWorker (render as JSON)
vmManagerStats,
// Array of room data (render as JSON)
currentRoomData,
// Arrays of last values (render as one column table)
vBrowserStartMS,
vBrowserStageRetries,
vBrowserStageFails,
vBrowserSessionMS,
// Maps of vbrowser users
vBrowserClientIDs,
vBrowserClientIDMinutes,
vBrowserUIDs,
vBrowserUIDMinutes,
};
}
function altArrayToObject(arr: string[] | undefined) {
const result: Record<string, number> = {};
if (!arr) {
return result;
}
for (let i = 0; i < arr.length; i += 2) {
const k = arr[i];
const v = arr[i + 1];
result[k] = Number(v);
}
return result;
}
+41
View File
@@ -0,0 +1,41 @@
import fs from "node:fs";
import { resolveShard } from "./resolveShard.ts";
let adjectives = fs
.readFileSync(process.cwd() + "/words/adjectives.txt")
.toString()
.split("\n");
const nouns = fs
.readFileSync(process.cwd() + "/words/nouns.txt")
.toString()
.split("\n");
const verbs = fs
.readFileSync(process.cwd() + "/words/verbs.txt")
.toString()
.split("\n");
const randomElement = (array: string[]) =>
array[Math.floor(Math.random() * array.length)];
export function makeRoomName(shard: number | undefined) {
let filteredAdjectives = adjectives;
if (shard) {
// Filter the adjective list by shard
filteredAdjectives = adjectives.filter(
(adj) => resolveShard(adj) === Number(shard),
);
}
const adjective = randomElement(filteredAdjectives);
const noun = randomElement(nouns);
const verb = randomElement(verbs);
return `${adjective}-${noun}-${verb}`;
}
export function makeUserName() {
return `${capFirst(randomElement(adjectives))} ${capFirst(
randomElement(nouns),
)}`;
}
function capFirst(string: string) {
return string.charAt(0).toUpperCase() + string.slice(1);
}
+7
View File
@@ -0,0 +1,7 @@
export const findPlaylistVideoByUrl = (
playlist: PlaylistVideo[],
url?: string,
) => {
if (!url) return;
return playlist.find((video) => video.url === url);
};
+87
View File
@@ -0,0 +1,87 @@
import { Client, type QueryResult } from "pg";
import config from "../config.ts";
export let postgres: Client | undefined = undefined;
if (config.DATABASE_URL) {
postgres = new Client({
connectionString: config.DATABASE_URL,
ssl: { rejectUnauthorized: false },
});
postgres.connect();
}
/**
* Use this if we need a new connection instead of sharing.
* Guarantees we'll return a client because we throw if we don't have it configured
* @returns
*/
export function newPostgres() {
if (!config.DATABASE_URL) {
throw new Error("postgres not configured");
}
const postgres = new Client({
connectionString: config.DATABASE_URL,
ssl: { rejectUnauthorized: false },
});
postgres.connect();
return postgres;
}
export async function updateObject(
postgres: Client,
table: string,
object: AnyDict,
condition: AnyDict,
): Promise<QueryResult<any>> {
const columns = Object.keys(object);
const values = Object.values(object);
// TODO support compound conditions, not just one
let query = `UPDATE ${table} SET ${columns
.map((c, i) => `"${c}" = $${i + 1}`)
.join(",")}
WHERE "${Object.keys(condition)[0]}" = $${Object.keys(object).length + 1}
RETURNING *`;
//console.log(query);
const result = await postgres.query(query, [
...values,
condition[Object.keys(condition)[0]],
]);
return result;
}
export async function insertObject(
postgres: Client,
table: string,
object: AnyDict,
): Promise<QueryResult<any>> {
const columns = Object.keys(object);
const values = Object.values(object);
let query = `INSERT INTO ${table} (${columns.map((c) => `"${c}"`).join(",")})
VALUES (${values.map((_, i) => "$" + (i + 1)).join(",")})
RETURNING *`;
// console.log(query);
const result = await postgres.query(query, values);
return result;
}
export async function upsertObject(
postgres: Client,
table: string,
object: AnyDict,
conflict: BooleanDict,
): Promise<QueryResult<any>> {
const columns = Object.keys(object);
const values = Object.values(object);
let query = `INSERT INTO ${table} (${columns.map((c) => `"${c}"`).join(",")})
VALUES (${values.map((_, i) => "$" + (i + 1)).join(",")})
ON CONFLICT (${Object.keys(conflict)
.map((k) => `"${k}"`)
.join(",")})
DO UPDATE SET ${Object.keys(object)
.map((c) => `"${c}" = EXCLUDED."${c}"`)
.join(",")}
RETURNING *`;
// console.log(query);
const result = await postgres.query(query, values);
return result;
}
+72
View File
@@ -0,0 +1,72 @@
import config from "../config.ts";
import { Redis } from "ioredis";
import { getStartOfHour } from "./time.ts";
export let redis: Redis | undefined = undefined;
if (config.REDIS_URL) {
redis = new Redis(config.REDIS_URL);
}
export async function redisCount(prefix: string) {
if (!redis) {
return;
}
try {
const key = `${prefix}:${getStartOfHour()}`;
await redis.incr(key);
await redis.expireat(key, getStartOfHour() + 86400 * 1000);
} catch (e) {
console.log(e);
}
}
export async function getRedisCountDay(prefix: string) {
if (!redis) {
return;
}
// Get counts for last 24 hour keys (including current partial hour)
const keyArr = [];
for (let i = 0; i < 24; i += 1) {
keyArr.push(`${prefix}:${getStartOfHour() - i * 3600 * 1000}`);
}
const values = await redis.mget(...keyArr);
return values.reduce((a, b) => (Number(a) || 0) + (Number(b) || 0), 0);
}
export async function getRedisCountHour(prefix: string) {
if (!redis) {
return;
}
// Get counts for previous full hour
const value = await redis.get(`${prefix}:${getStartOfHour() - 3600 * 1000}`);
return Number(value);
}
export async function redisCountDistinct(prefix: string, item: string) {
if (!redis) {
return;
}
const key = `${prefix}:${getStartOfHour()}`;
await redis.pfadd(key, item);
await redis.expireat(key, getStartOfHour() + 86400 * 1000);
}
export async function getRedisCountDayDistinct(prefix: string) {
if (!redis) {
return;
}
// Get counts for last 24 hour keys (including current partial hour)
const keyArr = [];
for (let i = 0; i < 24; i += 1) {
keyArr.push(`${prefix}:${getStartOfHour() - i * 3600 * 1000}`);
}
return await redis.pfcount(...keyArr);
}
export async function getRedisCountHourDistinct(prefix: string) {
if (!redis) {
return;
}
// Get counts for previous full hour
return await redis.pfcount(`${prefix}:${getStartOfHour() - 3600 * 1000}`);
}
+11
View File
@@ -0,0 +1,11 @@
/** This regex searches for YouTube Video IDs from a YouTube URL */
/** example: */
/** YOUTUBE_VIDEO_ID_REGEX.exec('https://youtube.com/?v=14634524364) */
/** will return the id 14634524364 in the first exec group [1] */
export const YOUTUBE_VIDEO_ID_REGEX =
/(?:youtube\.com\/(?:[^\/]+\/.+\/|(?:v|e(?:mbed)?)\/|.*[?&]v=)|youtu\.be\/)([^"&?\/\s]{11})/;
/** These regexes allow us to find hours, minutes and seconds from a ISO 8601 time string */
export const PT_HOURS_REGEX = /(\d{1,2})H/;
export const PT_MINUTES_REGEX = /(\d{1,2})M/;
export const PT_SECONDS_REGEX = /(\d{1,2})S/;
+12
View File
@@ -0,0 +1,12 @@
import config from "../config.ts";
import { apps } from "../ecosystem.config.js";
export function resolveShard(roomId: string): number {
if (!config.SHARD) {
return 0;
}
const numShards = apps.filter((app) => app.env?.SHARD).length;
const letter = roomId[0];
const charCode = letter.charCodeAt(0);
return Number((charCode % numShards) + 1);
}
+61
View File
@@ -0,0 +1,61 @@
import config from "../config.ts";
import Stripe from "stripe";
const stripe = new Stripe(config.STRIPE_SECRET_KEY, {
apiVersion: "2020-08-27",
});
export async function getCustomerByEmail(email: string) {
if (!config.STRIPE_SECRET_KEY) {
return undefined;
}
const customer = await stripe.customers.list({
email,
expand: ["data.subscriptions"],
});
return customer?.data[0];
}
export async function getIsSubscriberByEmail(email: string | undefined) {
if (!config.STRIPE_SECRET_KEY) {
// If Stripe isn't set up assume everyone is a subscriber
return true;
}
if (!email) {
return false;
}
const customer = await getCustomerByEmail(email);
const isSubscriber = Boolean(
customer?.subscriptions?.data?.find((sub) => sub?.status === "active"),
);
return isSubscriber;
}
export async function createSelfServicePortal(
customerId: string,
returnUrl: string,
) {
return await stripe.billingPortal.sessions.create({
customer: customerId,
return_url: returnUrl,
});
}
export async function getAllCustomers() {
const result = [];
for await (const customer of stripe.customers.list({ limit: 100 })) {
result.push(customer);
}
return result;
}
export async function getAllActiveSubscriptions() {
const result = [];
for await (const sub of stripe.subscriptions.list({
limit: 100,
status: "active",
})) {
result.push(sub);
}
return result;
}
+14
View File
@@ -0,0 +1,14 @@
export function getStartOfDay() {
const now = Date.now();
return now - (now % 86400000);
}
export function getStartOfHour() {
const now = Date.now();
return now - (now % 3600000);
}
export function getStartOfMinute() {
const now = Date.now();
return now - (now % 60000);
}
+127
View File
@@ -0,0 +1,127 @@
import config from "../config.ts";
import {
PT_HOURS_REGEX,
PT_MINUTES_REGEX,
PT_SECONDS_REGEX,
YOUTUBE_VIDEO_ID_REGEX,
} from "./regex.ts";
import { youtube, youtube_v3 } from "@googleapis/youtube";
let Youtube = config.YOUTUBE_API_KEY
? youtube({
version: "v3",
auth: config.YOUTUBE_API_KEY,
})
: null;
export const mapYoutubeSearchResult = (
video: youtube_v3.Schema$SearchResult,
): PlaylistVideo => {
return {
channel: video.snippet?.channelTitle ?? "",
url: "https://www.youtube.com/watch?v=" + video?.id?.videoId,
name: video.snippet?.title ?? "",
img: video.snippet?.thumbnails?.default?.url ?? "",
duration: 0,
type: "youtube",
};
};
export const mapYoutubeListResult = (
video: youtube_v3.Schema$Video,
): PlaylistVideo => {
const videoId = video.id;
return {
url: "https://www.youtube.com/watch?v=" + videoId,
name: video.snippet?.title ?? "",
img: video.snippet?.thumbnails?.default?.url ?? "",
channel: video.snippet?.channelTitle ?? "",
duration: getVideoDuration(video.contentDetails?.duration ?? ""),
type: "youtube",
};
};
export const mapYoutubePlaylistResult = (
item: youtube_v3.Schema$PlaylistItem,
): PlaylistVideo => {
return {
url: "https://www.youtube.com/watch?v=" + item.snippet?.resourceId?.videoId,
name: item.snippet?.title ?? "",
img: item.snippet?.thumbnails?.default?.url ?? "",
channel: item.snippet?.channelTitle ?? "",
duration: 0,
// duration: getVideoDuration(video.contentDetails?.duration ?? ''),
type: "youtube",
};
};
export const searchYoutube = async (
query: string,
): Promise<PlaylistVideo[]> => {
const response = await Youtube?.search.list({
part: ["snippet"],
type: ["video"],
maxResults: 25,
q: query,
});
return response?.data?.items?.map(mapYoutubeSearchResult) ?? [];
};
export const youtubePlaylist = async (
playlistId: string,
): Promise<PlaylistVideo[]> => {
const response = await Youtube?.playlistItems.list({
part: ["snippet"],
playlistId,
maxResults: 100,
});
return response?.data?.items?.map(mapYoutubePlaylistResult) ?? [];
};
export const getYoutubeVideoID = (url: string) => {
const idParts = YOUTUBE_VIDEO_ID_REGEX.exec(url);
if (!idParts) {
return;
}
const id = idParts[1];
if (!id) {
return;
}
return id;
};
export const fetchYoutubeVideo = async (
id: string,
): Promise<PlaylistVideo | null> => {
const response = await Youtube?.videos.list({
part: ["snippet", "contentDetails"],
id: [id],
});
const top = response?.data?.items?.[0];
return top ? mapYoutubeListResult(top) : null;
};
export const getVideoDuration = (string: string): number => {
if (!string) {
return 0;
}
const hoursParts = PT_HOURS_REGEX.exec(string);
const minutesParts = PT_MINUTES_REGEX.exec(string);
const secondsParts = PT_SECONDS_REGEX.exec(string);
const hours = hoursParts ? parseInt(hoursParts[1]) : 0;
const minutes = minutesParts ? parseInt(minutesParts[1]) : 0;
const seconds = secondsParts ? parseInt(secondsParts[1]) : 0;
const totalSeconds = seconds + minutes * 60 + hours * 60 * 60;
return totalSeconds;
};
export const isYouTube = (input: string) => {
return (
input.startsWith("https://www.youtube.com/") ||
input.startsWith("https://youtu.be/")
);
};