mirror of
https://github.com/FoxxMD/context-mod.git
synced 2026-04-19 03:00:07 -04:00
Add cache stat averages for key hits
This commit is contained in:
FoxxMD
@@ -1,4 +1,5 @@
|
||||
import {Duration} from "dayjs/plugin/duration";
|
||||
import {Cache} from 'cache-manager';
|
||||
import Poll from "snoostorm/out/util/Poll";
|
||||
import Snoowrap from "snoowrap";
|
||||
|
||||
@@ -1250,8 +1251,13 @@ export interface OperatorConfig extends OperatorJsonConfig {
|
||||
interface CacheTypeStat {
|
||||
requests: number,
|
||||
miss: number,
|
||||
missPercent?: string,
|
||||
identifierRequestCount: Cache
|
||||
identifierAverageHit: number
|
||||
requestTimestamps: number[]
|
||||
averageTimeBetweenHits: string
|
||||
}
|
||||
|
||||
export interface ResourceStats {
|
||||
[key: string]: CacheTypeStat
|
||||
[key: string]: CacheTypeStat;
|
||||
}
|
||||
|
||||
@@ -43,6 +43,10 @@ a {
|
||||
content: ":";
|
||||
}
|
||||
|
||||
.newRow {
|
||||
margin-top: 15px;
|
||||
}
|
||||
|
||||
.has-tooltip {
|
||||
/*position: relative;*/
|
||||
}
|
||||
|
||||
@@ -433,6 +433,8 @@ const rcbServer = function (options: OperatorConfig): ([() => Promise<void>, App
|
||||
Object.keys(curr.stats.cache.types as ResourceStats).forEach((k) => {
|
||||
acc[k].requests += curr.stats.cache.types[k].requests;
|
||||
acc[k].miss += curr.stats.cache.types[k].miss;
|
||||
acc[k].identifierAverageHit += Number.parseFloat(curr.stats.cache.types[k].identifierAverageHit);
|
||||
acc[k].averageTimeBetweenHits += curr.stats.cache.types[k].averageTimeBetweenHits === 'N/A' ? 0 : Number.parseFloat(curr.stats.cache.types[k].averageTimeBetweenHits)
|
||||
});
|
||||
return acc;
|
||||
}, cacheStats());
|
||||
@@ -440,6 +442,8 @@ const rcbServer = function (options: OperatorConfig): ([() => Promise<void>, App
|
||||
const per = acc[curr].miss === 0 ? 0 : formatNumber(acc[curr].miss / acc[curr].requests) * 100;
|
||||
// @ts-ignore
|
||||
acc[curr].missPercent = `${formatNumber(per, {toFixed: 0})}%`;
|
||||
acc[curr].identifierAverageHit = formatNumber(acc[curr].identifierAverageHit);
|
||||
acc[curr].averageTimeBetweenHits = formatNumber(acc[curr].averageTimeBetweenHits)
|
||||
return acc;
|
||||
}, cumRaw);
|
||||
const cacheReq = subManagerData.reduce((acc, curr) => acc + curr.stats.cache.totalRequests, 0);
|
||||
|
||||
@@ -475,40 +475,72 @@
|
||||
<span>
|
||||
- Results from <span class="font-mono">window</span> criteria.
|
||||
</span>
|
||||
<label>Author Criteria</label>
|
||||
<span><%= data.stats.cache.types.authorCrit.requests %> | <%= data.stats.cache.types.authorCrit.miss %> (<%= data.stats.cache.types.authorCrit.missPercent %>) miss</span>
|
||||
<label>Avgs</label>
|
||||
<span>Hits/Key <%= data.stats.cache.types.author.identifierAverageHit %> | Hit Interval <%= data.stats.cache.types.author.averageTimeBetweenHits %>s</span>
|
||||
<span>
|
||||
</span>
|
||||
<label class="newRow">Author Criteria</label>
|
||||
<span class="newRow"><%= data.stats.cache.types.authorCrit.requests %> | <%= data.stats.cache.types.authorCrit.miss %> (<%= data.stats.cache.types.authorCrit.missPercent %>) miss</span>
|
||||
<span class="newRow">
|
||||
- <span class="font-mono">authorIs</span> results
|
||||
</span>
|
||||
<label>Item Criteria</label>
|
||||
<span><%= data.stats.cache.types.itemCrit.requests %> | <%= data.stats.cache.types.itemCrit.miss %> (<%= data.stats.cache.types.itemCrit.missPercent %>) miss</span>
|
||||
<label>Avgs</label>
|
||||
<span>Hits/Key <%= data.stats.cache.types.authorCrit.identifierAverageHit %> | Hit Interval <%= data.stats.cache.types.authorCrit.averageTimeBetweenHits %>s</span>
|
||||
<span>
|
||||
</span>
|
||||
|
||||
<label class="newRow">Item Criteria</label>
|
||||
<span class="newRow"><%= data.stats.cache.types.itemCrit.requests %> | <%= data.stats.cache.types.itemCrit.miss %> (<%= data.stats.cache.types.itemCrit.missPercent %>) miss</span>
|
||||
<span class="newRow">
|
||||
- <span class="font-mono">itemIs</span> results
|
||||
</span>
|
||||
<label>Comment Check</label>
|
||||
<span><%= data.stats.cache.types.commentCheck.requests %> | <%= data.stats.cache.types.commentCheck.miss %> (<%= data.stats.cache.types.commentCheck.missPercent %>) miss</span>
|
||||
<label>Avgs</label>
|
||||
<span>Hits/Key <%= data.stats.cache.types.itemCrit.identifierAverageHit %> | Hit Interval <%= data.stats.cache.types.itemCrit.averageTimeBetweenHits %>s</span>
|
||||
<span>
|
||||
</span>
|
||||
|
||||
<label class="newRow">Comment Check</label>
|
||||
<span class="newRow"><%= data.stats.cache.types.commentCheck.requests %> | <%= data.stats.cache.types.commentCheck.miss %> (<%= data.stats.cache.types.commentCheck.missPercent %>) miss</span>
|
||||
<span class="newRow">
|
||||
- <span class="font-mono">cacheUserResult</span> results
|
||||
</span>
|
||||
<label>Submissions</label>
|
||||
<span><%= data.stats.cache.types.submission.requests %> | <%= data.stats.cache.types.submission.miss %> (<%= data.stats.cache.types.submission.missPercent %>) miss</span>
|
||||
<label>Avgs</label>
|
||||
<span>Hits/Key <%= data.stats.cache.types.commentCheck.identifierAverageHit %> | Hit Interval <%= data.stats.cache.types.commentCheck.averageTimeBetweenHits %>s</span>
|
||||
<span>
|
||||
|
||||
</span>
|
||||
<label>Comments</label>
|
||||
<span><%= data.stats.cache.types.comment.requests %> | <%= data.stats.cache.types.comment.miss %> (<%= data.stats.cache.types.comment.missPercent %>) miss</span>
|
||||
<span>
|
||||
|
||||
</span>
|
||||
<label>Content</label>
|
||||
<span><%= data.stats.cache.types.content.requests %> | <%= data.stats.cache.types.content.miss %> (<%= data.stats.cache.types.content.missPercent %>) miss</span>
|
||||
<label class="newRow">Submissions</label>
|
||||
<span class="newRow"><%= data.stats.cache.types.submission.requests %> | <%= data.stats.cache.types.submission.miss %> (<%= data.stats.cache.types.submission.missPercent %>) miss</span>
|
||||
<span class="newRow"></span>
|
||||
<label>Avgs</label>
|
||||
<span>Hits/Key <%= data.stats.cache.types.submission.identifierAverageHit %> | Hit Interval <%= data.stats.cache.types.submission.averageTimeBetweenHits %>s</span>
|
||||
<span>
|
||||
</span>
|
||||
|
||||
<label class="newRow">Comments</label>
|
||||
<span class="newRow"><%= data.stats.cache.types.comment.requests %> | <%= data.stats.cache.types.comment.miss %> (<%= data.stats.cache.types.comment.missPercent %>) miss</span>
|
||||
<span class="newRow"></span>
|
||||
<label>Avgs</label>
|
||||
<span>Hits/Key <%= data.stats.cache.types.comment.identifierAverageHit %> | Hit Interval <%= data.stats.cache.types.comment.averageTimeBetweenHits %>s</span>
|
||||
<span>
|
||||
</span>
|
||||
|
||||
<label class="newRow">Content</label>
|
||||
<span class="newRow"><%= data.stats.cache.types.content.requests %> | <%= data.stats.cache.types.content.miss %> (<%= data.stats.cache.types.content.missPercent %>) miss</span>
|
||||
<span class="newRow">
|
||||
- footer/comment/ban/message content.
|
||||
</span>
|
||||
<label>UserNote</label>
|
||||
<span><%= data.stats.cache.types.userNotes.requests %> | <%= data.stats.cache.types.userNotes.miss %> (<%= data.stats.cache.types.userNotes.missPercent %>) miss</span>
|
||||
<label>Avgs</label>
|
||||
<span>Hits/Key <%= data.stats.cache.types.content.identifierAverageHit %> | Hit Interval <%= data.stats.cache.types.content.averageTimeBetweenHits %>s</span>
|
||||
<span>
|
||||
</span>
|
||||
|
||||
<label class="newRow">UserNote</label>
|
||||
<span class="newRow"><%= data.stats.cache.types.userNotes.requests %> | <%= data.stats.cache.types.userNotes.miss %> (<%= data.stats.cache.types.userNotes.missPercent %>) miss</span>
|
||||
<span class="newRow"></span>
|
||||
<label>Avgs</label>
|
||||
<span>Hits/Key <%= data.stats.cache.types.userNotes.identifierAverageHit %> | Hit Interval <%= data.stats.cache.types.userNotes.averageTimeBetweenHits %>s</span>
|
||||
<span>
|
||||
</span>
|
||||
</div>
|
||||
</span>
|
||||
|
||||
@@ -166,7 +166,7 @@ export class Manager {
|
||||
};
|
||||
|
||||
if (this.resources !== undefined) {
|
||||
const resStats = this.resources.getStats();
|
||||
const resStats = await this.resources.getStats();
|
||||
|
||||
data.cache = resStats.cache;
|
||||
data.cache.currentKeyCount = await this.resources.getCacheKeyCount();
|
||||
|
||||
@@ -109,6 +109,7 @@ export class SubredditResources {
|
||||
};
|
||||
|
||||
const cacheUseCB = (miss: boolean) => {
|
||||
this.stats.cache.userNotes.requestTimestamps.push(Date.now());
|
||||
this.stats.cache.userNotes.requests++;
|
||||
this.stats.cache.userNotes.miss += miss ? 1 : 0;
|
||||
}
|
||||
@@ -135,23 +136,56 @@ export class SubredditResources {
|
||||
return 0;
|
||||
}
|
||||
|
||||
getStats() {
|
||||
async getStats() {
|
||||
const totals = Object.values(this.stats.cache).reduce((acc, curr) => ({
|
||||
miss: acc.miss + curr.miss,
|
||||
req: acc.req + curr.requests,
|
||||
}), {miss: 0, req: 0});
|
||||
const cacheKeys = Object.keys(this.stats.cache);
|
||||
return {
|
||||
cache: {
|
||||
// TODO could probably combine these two
|
||||
totalRequests: totals.req,
|
||||
totalMiss: totals.miss,
|
||||
missPercent: `${formatNumber(totals.miss === 0 || totals.req === 0 ? 0 :(totals.miss/totals.req) * 100, {toFixed: 0})}%`,
|
||||
types: Object.keys(this.stats.cache).reduce((acc, curr) => {
|
||||
types: await cacheKeys.reduce(async (accProm, curr) => {
|
||||
const acc = await accProm;
|
||||
// calculate miss percent
|
||||
|
||||
const per = acc[curr].miss === 0 ? 0 : formatNumber(acc[curr].miss / acc[curr].requests) * 100;
|
||||
// @ts-ignore
|
||||
acc[curr].missPercent = `${formatNumber(per, {toFixed: 0})}%`;
|
||||
|
||||
// calculate average identifier hits
|
||||
|
||||
const idCache = acc[curr].identifierRequestCount;
|
||||
// @ts-expect-error
|
||||
const idKeys = await idCache.store.keys() as string[];
|
||||
if(idKeys.length > 0) {
|
||||
let hits = 0;
|
||||
for (const k of idKeys) {
|
||||
hits += await idCache.get(k) as number;
|
||||
}
|
||||
acc[curr].identifierAverageHit = formatNumber(hits/idKeys.length);
|
||||
}
|
||||
|
||||
if(acc[curr].requestTimestamps.length > 1) {
|
||||
// calculate average time between request
|
||||
const diffData = acc[curr].requestTimestamps.reduce((acc, curr: number) => {
|
||||
if(acc.last === 0) {
|
||||
acc.last = curr;
|
||||
return acc;
|
||||
}
|
||||
acc.diffs.push(curr - acc.last);
|
||||
acc.last = curr;
|
||||
return acc;
|
||||
},{last: 0, diffs: [] as number[]});
|
||||
const avgDiff = diffData.diffs.reduce((acc, curr) => acc + curr, 0) / diffData.diffs.length;
|
||||
|
||||
acc[curr].averageTimeBetweenHits = formatNumber(avgDiff/1000);
|
||||
}
|
||||
|
||||
return acc;
|
||||
}, this.stats.cache)
|
||||
}, Promise.resolve(this.stats.cache))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -162,9 +196,13 @@ export class SubredditResources {
|
||||
|
||||
async getActivity(item: Submission | Comment) {
|
||||
try {
|
||||
let hash = '';
|
||||
if (item instanceof Submission && this.submissionTTL > 0) {
|
||||
hash = `sub-${item.name}`;
|
||||
await this.stats.cache.submission.identifierRequestCount.set(hash, (await this.stats.cache.submission.identifierRequestCount.wrap(hash, () => 0) as number) + 1);
|
||||
this.stats.cache.submission.requestTimestamps.push(Date.now());
|
||||
this.stats.cache.submission.requests++;
|
||||
const cachedSubmission = await this.cache.get(`sub-${item.name}`);
|
||||
const cachedSubmission = await this.cache.get(hash);
|
||||
if (cachedSubmission !== undefined) {
|
||||
this.logger.debug(`Cache Hit: Submission ${item.name}`);
|
||||
return cachedSubmission;
|
||||
@@ -172,11 +210,14 @@ export class SubredditResources {
|
||||
// @ts-ignore
|
||||
const submission = await item.fetch();
|
||||
this.stats.cache.submission.miss++;
|
||||
await this.cache.set(`sub-${item.name}`, submission, {ttl: this.submissionTTL});
|
||||
await this.cache.set(hash, submission, {ttl: this.submissionTTL});
|
||||
return submission;
|
||||
} else if (this.commentTTL > 0) {
|
||||
hash = `comm-${item.name}`;
|
||||
await this.stats.cache.comment.identifierRequestCount.set(hash, (await this.stats.cache.comment.identifierRequestCount.wrap(hash, () => 0) as number) + 1);
|
||||
this.stats.cache.comment.requestTimestamps.push(Date.now());
|
||||
this.stats.cache.comment.requests++;
|
||||
const cachedComment = await this.cache.get(`comm-${item.name}`);
|
||||
const cachedComment = await this.cache.get(hash);
|
||||
if (cachedComment !== undefined) {
|
||||
this.logger.debug(`Cache Hit: Comment ${item.name}`);
|
||||
return cachedComment;
|
||||
@@ -184,7 +225,7 @@ export class SubredditResources {
|
||||
// @ts-ignore
|
||||
const comment = await item.fetch();
|
||||
this.stats.cache.comment.miss++;
|
||||
await this.cache.set(`comm-${item.name}`, comment, {ttl: this.commentTTL});
|
||||
await this.cache.set(hash, comment, {ttl: this.commentTTL});
|
||||
return comment;
|
||||
} else {
|
||||
// @ts-ignore
|
||||
@@ -206,6 +247,8 @@ export class SubredditResources {
|
||||
const hash = objectHash.sha1({...options, userName});
|
||||
|
||||
this.stats.cache.author.requests++;
|
||||
await this.stats.cache.author.identifierRequestCount.set(user.name, (await this.stats.cache.author.identifierRequestCount.wrap(user.name, () => 0) as number) + 1);
|
||||
this.stats.cache.author.requestTimestamps.push(Date.now());
|
||||
let miss = false;
|
||||
const cacheVal = await this.cache.wrap(hash, async () => {
|
||||
miss = true;
|
||||
@@ -251,6 +294,8 @@ export class SubredditResources {
|
||||
// try to get cached value first
|
||||
let hash = `${subreddit.display_name}-${cacheKey}`;
|
||||
if (this.wikiTTL > 0) {
|
||||
await this.stats.cache.content.identifierRequestCount.set(cacheKey, (await this.stats.cache.content.identifierRequestCount.wrap(cacheKey, () => 0) as number) + 1);
|
||||
this.stats.cache.content.requestTimestamps.push(Date.now());
|
||||
this.stats.cache.content.requests++;
|
||||
const cachedContent = await this.cache.get(hash);
|
||||
if (cachedContent !== undefined) {
|
||||
@@ -311,6 +356,8 @@ export class SubredditResources {
|
||||
if (this.filterCriteriaTTL > 0) {
|
||||
const hashObj = {itemId: item.id, ...authorOpts, include};
|
||||
const hash = `authorCrit-${objectHash.sha1(hashObj)}`;
|
||||
await this.stats.cache.authorCrit.identifierRequestCount.set(hash, (await this.stats.cache.authorCrit.identifierRequestCount.wrap(hash, () => 0) as number) + 1);
|
||||
this.stats.cache.authorCrit.requestTimestamps.push(Date.now());
|
||||
this.stats.cache.authorCrit.requests++;
|
||||
let miss = false;
|
||||
const cachedAuthorTest = await this.cache.wrap(hash, async () => {
|
||||
@@ -345,6 +392,8 @@ export class SubredditResources {
|
||||
try {
|
||||
const hashObj = {itemId: item.name, ...states};
|
||||
const hash = `itemCrit-${objectHash.sha1(hashObj)}`;
|
||||
await this.stats.cache.itemCrit.identifierRequestCount.set(hash, (await this.stats.cache.itemCrit.identifierRequestCount.wrap(hash, () => 0) as number) + 1);
|
||||
this.stats.cache.itemCrit.requestTimestamps.push(Date.now());
|
||||
this.stats.cache.itemCrit.requests++;
|
||||
const cachedItem = await this.cache.get(hash);
|
||||
if (cachedItem !== undefined) {
|
||||
|
||||
26
src/util.ts
26
src/util.ts
@@ -876,16 +876,26 @@ export const removeUndefinedKeys = (obj: any) => {
|
||||
return newObj;
|
||||
}
|
||||
|
||||
const timestampArr = () => {
|
||||
const arr: number[] = [];
|
||||
arr.length = 20;
|
||||
return arr;
|
||||
}
|
||||
|
||||
const statMetricCache = () => {
|
||||
return cacheManager.caching({store: 'memory', max: 50, ttl: 0});
|
||||
}
|
||||
|
||||
export const cacheStats = (): ResourceStats => {
|
||||
return {
|
||||
author: {requests: 0, miss: 0},
|
||||
authorCrit: {requests: 0, miss: 0},
|
||||
itemCrit: {requests: 0, miss: 0},
|
||||
content: {requests: 0, miss: 0},
|
||||
userNotes: {requests: 0, miss: 0},
|
||||
submission: {requests: 0, miss: 0},
|
||||
comment: {requests: 0, miss: 0},
|
||||
commentCheck: {requests: 0, miss: 0}
|
||||
author: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
|
||||
authorCrit: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
|
||||
itemCrit: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
|
||||
content: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
|
||||
userNotes: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
|
||||
submission: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
|
||||
comment: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0},
|
||||
commentCheck: {requests: 0, miss: 0, identifierRequestCount: statMetricCache(), requestTimestamps: timestampArr(), averageTimeBetweenHits: 'N/A', identifierAverageHit: 0}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user