discourse/discourse-ai
1
0
Fork 0
mirror of https://github.com/discourse/discourse-ai.git synced 2025-06-25 09:02:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

FEATURE: Add spending metrics to AI usage (#1268)

Browse Source 
This update adds metrics for estimated spending in AI usage. To make use of it, admins must add cost details to the LLM config page (input, output, and cached input costs per 1M tokens). After doing so, the metrics will appear in the AI usage dashboard as the AI plugin is used.
This commit is contained in:
Keegan George 2025-04-17 15:09:48 -07:00 committed by GitHub
parent e2b0287333
commit d26c7ac48d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 559 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-llms-edit.js
View File

@ -2,8 +2,17 @@ import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiLlmsEdit extends DiscourseRoute {
async model(params) {
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
const id = parseInt(params.id, 10);
if (id < 0) {
// You shouldn't be able to access the edit page
// if the model is seeded
return this.router.transitionTo(
"adminPlugins.show.discourse-ai-llms.index"
);
}
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
const record = allLlms.findBy("id", id);
record.provider_params = record.provider_params || {};
return record;

3
app/controllers/discourse_ai/admin/ai_llms_controller.rb
View File

@ -161,6 +161,9 @@ module DiscourseAi
:api_key,
:enabled_chat_bot,
:vision_enabled,
:input_cost,
:cached_input_cost,
:output_cost,
)
provider = updating ? updating.provider : permitted[:provider]

12
app/models/llm_model.rb
View File

@ -13,7 +13,14 @@ class LlmModel < ActiveRecord::Base
validates :url, presence: true, unless: -> { provider == BEDROCK_PROVIDER_NAME }
validates_presence_of :name, :api_key
validates :max_prompt_tokens, numericality: { greater_than: 0 }
validates :max_output_tokens, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true
validates :input_cost,
:cached_input_cost,
:output_cost,
:max_output_tokens,
numericality: {
greater_than_or_equal_to: 0,
},
allow_nil: true
validate :required_provider_params
scope :in_use,
-> do
@ -184,5 +191,8 @@ end
# enabled_chat_bot :boolean default(FALSE), not null
# provider_params :jsonb
# vision_enabled :boolean default(FALSE), not null
# input_cost :float
# cached_input_cost :float
# output_cost :float
# max_output_tokens :integer
#

10
app/serializers/ai_usage_serializer.rb
View File

@ -22,6 +22,9 @@ class AiUsageSerializer < ApplicationSerializer
total_cached_tokens
total_request_tokens
total_response_tokens
input_spending
output_spending
cached_input_spending
],
)
end
@ -35,6 +38,9 @@ class AiUsageSerializer < ApplicationSerializer
total_cached_tokens
total_request_tokens
total_response_tokens
input_spending
output_spending
cached_input_spending
],
)
end
@ -49,6 +55,9 @@ class AiUsageSerializer < ApplicationSerializer
total_cached_tokens: user.total_cached_tokens,
total_request_tokens: user.total_request_tokens,
total_response_tokens: user.total_response_tokens,
input_spending: user.input_spending,
output_spending: user.output_spending,
cached_input_spending: user.cached_input_spending,
}
end
end
@ -60,6 +69,7 @@ class AiUsageSerializer < ApplicationSerializer
total_request_tokens: object.total_request_tokens,
total_response_tokens: object.total_response_tokens,
total_requests: object.total_requests,
total_spending: object.total_spending,
date_range: {
start: object.start_date,
end: object.end_date,

3
app/serializers/llm_model_serializer.rb
View File

@ -18,6 +18,9 @@ class LlmModelSerializer < ApplicationSerializer
:enabled_chat_bot,
:provider_params,
:vision_enabled,
:input_cost,
:output_cost,
:cached_input_cost,
:used_by
has_one :user, serializer: BasicUserSerializer, embed: :object

5
assets/javascripts/discourse/admin/models/ai-llm.js
View File

@ -15,7 +15,10 @@ export default class AiLlm extends RestModel {
"api_key",
"enabled_chat_bot",
"provider_params",
"vision_enabled"
"vision_enabled",
"input_cost",
"cached_input_cost",
"output_cost"
);
}

512
assets/javascripts/discourse/components/ai-llm-editor-form.gjs
View File

@ -47,6 +47,9 @@ export default class AiLlmEditorForm extends Component {
name: modelInfo.name,
provider: info.provider,
provider_params: this.computeProviderParams(info.provider),
input_cost: modelInfo.input_cost,
output_cost: modelInfo.output_cost,
cached_input_cost: modelInfo.cached_input_cost,
};
}
@ -63,6 +66,9 @@ export default class AiLlmEditorForm extends Component {
provider: model.provider,
enabled_chat_bot: model.enabled_chat_bot,
vision_enabled: model.vision_enabled,
input_cost: model.input_cost,
output_cost: model.output_cost,
cached_input_cost: model.cached_input_cost,
provider_params: this.computeProviderParams(
model.provider,
model.provider_params
@ -118,10 +124,6 @@ export default class AiLlmEditorForm extends Component {
return localized.join(", ");
}
get seeded() {
return this.args.model.id < 0;
}
get inUseWarning() {
return i18n("discourse_ai.llms.in_use_warning", {
settings: this.modulesUsingModel,
@ -271,15 +273,9 @@ export default class AiLlmEditorForm extends Component {
<Form
@onSubmit={{this.save}}
@data={{this.formData}}
class="ai-llm-editor {{if this.seeded 'seeded'}}"
class="ai-llm-editor"
as |form data|
>
{{#if this.seeded}}
<form.Alert @icon="circle-info">
{{i18n "discourse_ai.llms.seeded_warning"}}
</form.Alert>
{{/if}}
{{#if this.modulesUsingModel}}
<form.Alert @icon="circle-info">
{{this.inUseWarning}}
@ -290,7 +286,6 @@ export default class AiLlmEditorForm extends Component {
@name="display_name"
@title={{i18n "discourse_ai.llms.display_name"}}
@validation="required|length:1,100"
@disabled={{this.seeded}}
@format="large"
@tooltip={{i18n "discourse_ai.llms.hints.display_name"}}
as |field|
@ -303,7 +298,6 @@ export default class AiLlmEditorForm extends Component {
@title={{i18n "discourse_ai.llms.name"}}
@tooltip={{i18n "discourse_ai.llms.hints.name"}}
@validation="required"
@disabled={{this.seeded}}
@format="large"
as |field|
>
@ -313,7 +307,6 @@ export default class AiLlmEditorForm extends Component {
<form.Field
@name="provider"
@title={{i18n "discourse_ai.llms.provider"}}
@disabled={{this.seeded}}
@format="large"
@validation="required"
@onSet={{this.setProvider}}
@ -328,262 +321,289 @@ export default class AiLlmEditorForm extends Component {
</field.Select>
</form.Field>
{{#unless this.seeded}}
{{#if (this.canEditURL data.provider)}}
<form.Field
@name="url"
@title={{i18n "discourse_ai.llms.url"}}
@validation="required"
@format="large"
as |field|
>
<field.Input />
</form.Field>
{{/if}}
{{#if (this.canEditURL data.provider)}}
<form.Field
@name="api_key"
@title={{i18n "discourse_ai.llms.api_key"}}
@name="url"
@title={{i18n "discourse_ai.llms.url"}}
@validation="required"
@format="large"
as |field|
>
<field.Password />
<field.Input />
</form.Field>
{{/if}}
<form.Object @name="provider_params" as |object providerParamsData|>
{{#each (this.providerParamsKeys providerParamsData) as |name|}}
{{#let
(get (this.metaProviderParams data.provider) name)
as |params|
}}
<object.Field
@name={{name}}
@title={{i18n
(concat "discourse_ai.llms.provider_fields." name)
}}
@format="large"
as |field|
>
{{#if (eq params.type "enum")}}
<field.Select @includeNone={{false}} as |select|>
{{#each params.values as |option|}}
<select.Option
@value={{option.id}}
>{{option.name}}</select.Option>
{{/each}}
</field.Select>
{{else if (eq params.type "checkbox")}}
<field.Checkbox />
{{else}}
<field.Input @type={{params.type}} />
{{/if}}
</object.Field>
{{/let}}
{{/each}}
</form.Object>
<form.Field
@name="api_key"
@title={{i18n "discourse_ai.llms.api_key"}}
@validation="required"
@format="large"
as |field|
>
<field.Password />
</form.Field>
<form.Field
@name="tokenizer"
@title={{i18n "discourse_ai.llms.tokenizer"}}
@disabled={{this.seeded}}
@format="large"
@validation="required"
as |field|
>
<field.Select as |select|>
{{#each this.tokenizers as |tokenizer|}}
<select.Option
@value={{tokenizer.id}}
>{{tokenizer.name}}</select.Option>
{{/each}}
</field.Select>
</form.Field>
<form.Field
@name="max_prompt_tokens"
@title={{i18n "discourse_ai.llms.max_prompt_tokens"}}
@tooltip={{i18n "discourse_ai.llms.hints.max_prompt_tokens"}}
@validation="required"
@format="large"
as |field|
>
<field.Input @type="number" step="any" min="0" lang="en" />
</form.Field>
<form.Field
@name="max_output_tokens"
@title={{i18n "discourse_ai.llms.max_output_tokens"}}
@tooltip={{i18n "discourse_ai.llms.hints.max_output_tokens"}}
@format="large"
as |field|
>
<field.Input @type="number" step="any" min="0" lang="en" />
</form.Field>
<form.Field
@name="vision_enabled"
@title={{i18n "discourse_ai.llms.vision_enabled"}}
@tooltip={{i18n "discourse_ai.llms.hints.vision_enabled"}}
@format="large"
as |field|
>
<field.Checkbox />
</form.Field>
<form.Field
@name="enabled_chat_bot"
@title={{i18n "discourse_ai.llms.enabled_chat_bot"}}
@tooltip={{i18n "discourse_ai.llms.hints.enabled_chat_bot"}}
@format="large"
as |field|
>
<field.Checkbox />
</form.Field>
{{#if @model.user}}
<form.Container @title={{i18n "discourse_ai.llms.ai_bot_user"}}>
<a
class="avatar"
href={{@model.user.path}}
data-user-card={{@model.user.username}}
<form.Object @name="provider_params" as |object providerParamsData|>
{{#each (this.providerParamsKeys providerParamsData) as |name|}}
{{#let
(get (this.metaProviderParams data.provider) name)
as |params|
}}
<object.Field
@name={{name}}
@title={{i18n (concat "discourse_ai.llms.provider_fields." name)}}
@format="large"
as |field|
>
{{Avatar @model.user.avatar_template "small"}}
</a>
<LinkTo @route="adminUser" @model={{this.adminUser}}>
{{@model.user.username}}
</LinkTo>
</form.Container>
{{/if}}
{{#if (eq params.type "enum")}}
<field.Select @includeNone={{false}} as |select|>
{{#each params.values as |option|}}
<select.Option
@value={{option.id}}
>{{option.name}}</select.Option>
{{/each}}
</field.Select>
{{else if (eq params.type "checkbox")}}
<field.Checkbox />
{{else}}
<field.Input @type={{params.type}} />
{{/if}}
</object.Field>
{{/let}}
{{/each}}
</form.Object>
{{#if (gt data.llm_quotas.length 0)}}
<form.Container @title={{i18n "discourse_ai.llms.quotas.title"}}>
<table class="ai-llm-quotas__table">
<thead class="ai-llm-quotas__table-head">
<tr class="ai-llm-quotas__header-row">
<th class="ai-llm-quotas__header">{{i18n
"discourse_ai.llms.quotas.group"
}}</th>
<th class="ai-llm-quotas__header">{{i18n
"discourse_ai.llms.quotas.max_tokens"
}}</th>
<th class="ai-llm-quotas__header">{{i18n
"discourse_ai.llms.quotas.max_usages"
}}</th>
<th class="ai-llm-quotas__header">{{i18n
"discourse_ai.llms.quotas.duration"
}}</th>
<th
class="ai-llm-quotas__header ai-llm-quotas__header--actions"
></th>
<th></th>
</tr>
</thead>
<tbody class="ai-llm-quotas__table-body">
<form.Collection
@name="llm_quotas"
@tagName="tr"
class="ai-llm-quotas__row"
as |collection index collectionData|
>
<td
class="ai-llm-quotas__cell"
>{{collectionData.group_name}}</td>
<td class="ai-llm-quotas__cell">
<collection.Field
@name="max_tokens"
@title="max_tokens"
@showTitle={{false}}
as |field|
>
<field.Input
@type="number"
class="ai-llm-quotas__input"
min="1"
/>
</collection.Field>
</td>
<td class="ai-llm-quotas__cell">
<collection.Field
@name="max_usages"
@title="max_usages"
@showTitle={{false}}
as |field|
>
<field.Input
@type="number"
class="ai-llm-quotas__input"
min="1"
/>
</collection.Field>
</td>
<td class="ai-llm-quotas__cell">
<collection.Field
@name="duration_seconds"
@title="duration_seconds"
@showTitle={{false}}
as |field|
>
<field.Custom>
<DurationSelector
@value={{collectionData.duration_seconds}}
@onChange={{field.set}}
/>
</field.Custom>
</collection.Field>
</td>
<td>
<form.Button
@icon="trash-can"
@action={{fn collection.remove index}}
class="btn-danger ai-llm-quotas__delete-btn"
<form.Field
@name="tokenizer"
@title={{i18n "discourse_ai.llms.tokenizer"}}
@format="large"
@validation="required"
as |field|
>
<field.Select as |select|>
{{#each this.tokenizers as |tokenizer|}}
<select.Option
@value={{tokenizer.id}}
>{{tokenizer.name}}</select.Option>
{{/each}}
</field.Select>
</form.Field>
<form.Field
@name="max_prompt_tokens"
@title={{i18n "discourse_ai.llms.max_prompt_tokens"}}
@tooltip={{i18n "discourse_ai.llms.hints.max_prompt_tokens"}}
@validation="required"
@format="large"
as |field|
>
<field.Input @type="number" step="any" min="0" lang="en" />
</form.Field>
<form.InputGroup as |inputGroup|>
<inputGroup.Field
@name="input_cost"
@title={{i18n "discourse_ai.llms.cost_input"}}
@tooltip={{i18n "discourse_ai.llms.hints.cost_input"}}
@helpText={{i18n "discourse_ai.llms.hints.cost_measure"}}
as |field|
>
<field.Input @type="number" step="any" min="0" lang="en" />
</inputGroup.Field>
<inputGroup.Field
@name="cached_input_cost"
@title={{i18n "discourse_ai.llms.cost_cached_input"}}
@tooltip={{i18n "discourse_ai.llms.hints.cost_cached_input"}}
@helpText={{i18n "discourse_ai.llms.hints.cost_measure"}}
as |field|
>
<field.Input @type="number" step="any" min="0" lang="en" />
</inputGroup.Field>
<inputGroup.Field
@name="output_cost"
@title={{i18n "discourse_ai.llms.cost_output"}}
@tooltip={{i18n "discourse_ai.llms.hints.cost_output"}}
@helpText={{i18n "discourse_ai.llms.hints.cost_measure"}}
as |field|
>
<field.Input @type="number" step="any" min="0" lang="en" />
</inputGroup.Field>
</form.InputGroup>
<form.Field
@name="max_output_tokens"
@title={{i18n "discourse_ai.llms.max_output_tokens"}}
@tooltip={{i18n "discourse_ai.llms.hints.max_output_tokens"}}
@format="large"
as |field|
>
<field.Input @type="number" step="any" min="0" lang="en" />
</form.Field>
<form.Field
@name="vision_enabled"
@title={{i18n "discourse_ai.llms.vision_enabled"}}
@tooltip={{i18n "discourse_ai.llms.hints.vision_enabled"}}
@format="large"
as |field|
>
<field.Checkbox />
</form.Field>
<form.Field
@name="enabled_chat_bot"
@title={{i18n "discourse_ai.llms.enabled_chat_bot"}}
@tooltip={{i18n "discourse_ai.llms.hints.enabled_chat_bot"}}
@format="large"
as |field|
>
<field.Checkbox />
</form.Field>
{{#if @model.user}}
<form.Container @title={{i18n "discourse_ai.llms.ai_bot_user"}}>
<a
class="avatar"
href={{@model.user.path}}
data-user-card={{@model.user.username}}
>
{{Avatar @model.user.avatar_template "small"}}
</a>
<LinkTo @route="adminUser" @model={{this.adminUser}}>
{{@model.user.username}}
</LinkTo>
</form.Container>
{{/if}}
{{#if (gt data.llm_quotas.length 0)}}
<form.Container @title={{i18n "discourse_ai.llms.quotas.title"}}>
<table class="ai-llm-quotas__table">
<thead class="ai-llm-quotas__table-head">
<tr class="ai-llm-quotas__header-row">
<th class="ai-llm-quotas__header">{{i18n
"discourse_ai.llms.quotas.group"
}}</th>
<th class="ai-llm-quotas__header">{{i18n
"discourse_ai.llms.quotas.max_tokens"
}}</th>
<th class="ai-llm-quotas__header">{{i18n
"discourse_ai.llms.quotas.max_usages"
}}</th>
<th class="ai-llm-quotas__header">{{i18n
"discourse_ai.llms.quotas.duration"
}}</th>
<th
class="ai-llm-quotas__header ai-llm-quotas__header--actions"
></th>
<th></th>
</tr>
</thead>
<tbody class="ai-llm-quotas__table-body">
<form.Collection
@name="llm_quotas"
@tagName="tr"
class="ai-llm-quotas__row"
as |collection index collectionData|
>
<td
class="ai-llm-quotas__cell"
>{{collectionData.group_name}}</td>
<td class="ai-llm-quotas__cell">
<collection.Field
@name="max_tokens"
@title="max_tokens"
@showTitle={{false}}
as |field|
>
<field.Input
@type="number"
class="ai-llm-quotas__input"
min="1"
/>
</td>
</form.Collection>
</tbody>
</table>
</form.Container>
</collection.Field>
</td>
<td class="ai-llm-quotas__cell">
<collection.Field
@name="max_usages"
@title="max_usages"
@showTitle={{false}}
as |field|
>
<field.Input
@type="number"
class="ai-llm-quotas__input"
min="1"
/>
</collection.Field>
</td>
<td class="ai-llm-quotas__cell">
<collection.Field
@name="duration_seconds"
@title="duration_seconds"
@showTitle={{false}}
as |field|
>
<field.Custom>
<DurationSelector
@value={{collectionData.duration_seconds}}
@onChange={{field.set}}
/>
</field.Custom>
</collection.Field>
</td>
<td>
<form.Button
@icon="trash-can"
@action={{fn collection.remove index}}
class="btn-danger ai-llm-quotas__delete-btn"
/>
</td>
</form.Collection>
</tbody>
</table>
</form.Container>
<form.Button
@action={{fn
this.openAddQuotaModal
(fn form.addItemToCollection "llm_quotas")
}}
@icon="plus"
@label="discourse_ai.llms.quotas.add"
class="ai-llm-editor__add-quota-btn"
/>
{{/if}}
<form.Actions>
<form.Button
@action={{fn this.test data}}
@disabled={{this.testRunning}}
@label="discourse_ai.llms.tests.title"
/>
<form.Submit />
{{#if (eq data.llm_quotas.length 0)}}
<form.Button
@action={{fn
this.openAddQuotaModal
(fn form.addItemToCollection "llm_quotas")
}}
@icon="plus"
@label="discourse_ai.llms.quotas.add"
class="ai-llm-editor__add-quota-btn"
/>
{{/if}}
<form.Actions>
{{#unless @model.isNew}}
<form.Button
@action={{fn this.test data}}
@disabled={{this.testRunning}}
@label="discourse_ai.llms.tests.title"
@action={{this.delete}}
@label="discourse_ai.llms.delete"
class="btn-danger"
/>
<form.Submit />
{{#if (eq data.llm_quotas.length 0)}}
<form.Button
@action={{fn
this.openAddQuotaModal
(fn form.addItemToCollection "llm_quotas")
}}
@label="discourse_ai.llms.quotas.add"
class="ai-llm-editor__add-quota-btn"
/>
{{/if}}
{{#unless @model.isNew}}
<form.Button
@action={{this.delete}}
@label="discourse_ai.llms.delete"
class="btn-danger"
/>
{{/unless}}
</form.Actions>
{{/unless}}
{{/unless}}
</form.Actions>
{{#if this.displayTestResult}}
<form.Container @format="full">

61
assets/javascripts/discourse/components/ai-usage.gjs
View File

@ -2,6 +2,8 @@ import Component from "@glimmer/component";
import { tracked } from "@glimmer/tracking";
import { fn, hash } from "@ember/helper";
import { action } from "@ember/object";
import didInsert from "@ember/render-modifiers/modifiers/did-insert";
import didUpdate from "@ember/render-modifiers/modifiers/did-update";
import { LinkTo } from "@ember/routing";
import { service } from "@ember/service";
import { eq, gt, lt } from "truth-helpers";
@ -74,6 +76,22 @@ export default class AiUsage extends Component {
this.onFilterChange();
}
@action
addCurrencyChar(element) {
element.querySelectorAll(".d-stat-tile__label").forEach((label) => {
if (
label.innerText.trim() === i18n("discourse_ai.usage.total_spending")
) {
const valueElement = label
.closest(".d-stat-tile")
?.querySelector(".d-stat-tile__value");
if (valueElement) {
valueElement.innerText = `$${valueElement.innerText}`;
}
}
});
}
@bind
takeUsers(start, end) {
return this.data.users.slice(start, end);
@ -153,6 +171,11 @@ export default class AiUsage extends Component {
value: this.data.summary.total_cached_tokens,
tooltip: i18n("discourse_ai.usage.stat_tooltips.cached_tokens"),
},
{
label: i18n("discourse_ai.usage.total_spending"),
value: this.data.summary.total_spending,
tooltip: i18n("discourse_ai.usage.stat_tooltips.total_spending"),
},
];
}
@ -308,6 +331,11 @@ export default class AiUsage extends Component {
this.fetchData();
}
totalSpending(inputSpending, cachedSpending, outputSpending) {
const total = inputSpending + cachedSpending + outputSpending;
return `$${total.toFixed(2)}`;
}
<template>
<div class="ai-usage admin-detail">
<DPageSubheader
@ -376,9 +404,15 @@ export default class AiUsage extends Component {
class="ai-usage__summary"
>
<:content>
<DStatTiles as |tiles|>
<DStatTiles
{{didInsert this.addCurrencyChar this.metrics}}
{{didUpdate this.addCurrencyChar this.metrics}}
as |tiles|
>
{{#each this.metrics as |metric|}}
<tiles.Tile
class="bar"
@label={{metric.label}}
@href={{metric.href}}
@value={{metric.value}}
@ -422,6 +456,7 @@ export default class AiUsage extends Component {
<th>{{i18n "discourse_ai.usage.feature"}}</th>
<th>{{i18n "discourse_ai.usage.usage_count"}}</th>
<th>{{i18n "discourse_ai.usage.total_tokens"}}</th>
<th>{{i18n "discourse_ai.usage.total_spending"}}</th>
</tr>
</thead>
<tbody>
@ -438,6 +473,13 @@ export default class AiUsage extends Component {
class="ai-usage__features-cell"
title={{feature.total_tokens}}
>{{number feature.total_tokens}}</td>
<td>
{{this.totalSpending
feature.input_spending
feature.cached_input_spending
feature.output_spending
}}
</td>
</tr>
{{/each}}
</tbody>
@ -464,6 +506,8 @@ export default class AiUsage extends Component {
<th>{{i18n "discourse_ai.usage.model"}}</th>
<th>{{i18n "discourse_ai.usage.usage_count"}}</th>
<th>{{i18n "discourse_ai.usage.total_tokens"}}</th>
<th>{{i18n "discourse_ai.usage.total_spending"}}</th>
</tr>
</thead>
<tbody>
@ -478,6 +522,13 @@ export default class AiUsage extends Component {
class="ai-usage__models-cell"
title={{model.total_tokens}}
>{{number model.total_tokens}}</td>
<td>
{{this.totalSpending
model.input_spending
model.cached_input_spending
model.output_spending
}}
</td>
</tr>
{{/each}}
</tbody>
@ -511,6 +562,7 @@ export default class AiUsage extends Component {
}}</th>
<th>{{i18n "discourse_ai.usage.usage_count"}}</th>
<th>{{i18n "discourse_ai.usage.total_tokens"}}</th>
<th>{{i18n "discourse_ai.usage.total_spending"}}</th>
</tr>
</thead>
<tbody>
@ -535,6 +587,13 @@ export default class AiUsage extends Component {
class="ai-usage__users-cell"
title={{user.total_tokens}}
>{{number user.total_tokens}}</td>
<td>
{{this.totalSpending
user.input_spending
user.cached_input_spending
user.output_spending
}}
</td>
</tr>
{{/each}}
</tbody>

10
config/locales/client.en.yml
View File

@ -239,6 +239,7 @@ en:
net_request_tokens: "Net request tokens"
cached_tokens: "Cached tokens"
cached_request_tokens: "Cached request tokens"
total_spending: "Estimated cost"
no_users: "No user usage data found"
no_models: "No model usage data found"
no_features: "No feature usage data found"
@ -249,6 +250,7 @@ en:
request_tokens: "Tokens used when the LLM tries to understand what you are saying"
response_tokens: "Tokens used when the LLM responds to your prompt"
cached_tokens: "Previously processed request tokens that the LLM reuses to optimize performance and cost"
total_spending: "Cumulative cost of all tokens used by the LLMs based on specified cost metrics added to LLM configuration settings"
periods:
last_day: "Last 24 hours"
last_week: "Last week"
@ -404,6 +406,10 @@ en:
enabled_chat_bot: "Allow AI bot selector"
vision_enabled: "Vision enabled"
ai_bot_user: "AI bot User"
cost_input: "Input cost"
cost_cached_input: "Cached input cost"
cost_output: "Output cost"
save: "Save"
edit: "Edit"
saved: "LLM model saved"
@ -487,6 +493,10 @@ en:
name: "We include this in the API call to specify which model we'll use"
vision_enabled: "If enabled, the AI will attempt to understand images. It depends on the model being used supporting vision. Supported by latest models from Anthropic, Google, and OpenAI."
enabled_chat_bot: "If enabled, users can select this model when creating PMs with the AI bot"
cost_input: "The input cost per 1M tokens for this model"
cost_cached_input: "The cached input cost per 1M tokens for this model"
cost_output: "The output cost per 1M tokens for this model"
cost_measure: "$/1M tokens"
providers:
aws_bedrock: "AWS Bedrock"
anthropic: "Anthropic"

9
db/migrate/20250416215039_add_cost_metrics_to_llm_model.rb Normal file
View File

@ -0,0 +1,9 @@
# frozen_string_literal: true
class AddCostMetricsToLlmModel < ActiveRecord::Migration[7.2]
def change
add_column :llm_models, :input_cost, :float
add_column :llm_models, :cached_input_cost, :float
add_column :llm_models, :output_cost, :float
end
end

72
lib/completions/llm.rb
View File

@ -30,9 +30,26 @@ module DiscourseAi
name: "claude-3-7-sonnet",
tokens: 200_000,
display_name: "Claude 3.7 Sonnet",
input_cost: 3,
cached_input_cost: 0.30,
output_cost: 15,
},
{
name: "claude-3-5-haiku",
tokens: 200_000,
display_name: "Claude 3.5 Haiku",
input_cost: 0.80,
cached_input_cost: 0.08,
output_cost: 4,
},
{
name: "claude-3-opus",
tokens: 200_000,
display_name: "Claude 3 Opus",
input_cost: 15,
cached_input_cost: 1.50,
output_cost: 75,
},
{ name: "claude-3-5-haiku", tokens: 200_000, display_name: "Claude 3.5 Haiku" },
{ name: "claude-3-opus", tokens: 200_000, display_name: "Claude 3 Opus" },
],
tokenizer: DiscourseAi::Tokenizer::AnthropicTokenizer,
endpoint: "https://api.anthropic.com/v1/messages",
@ -61,6 +78,8 @@ module DiscourseAi
endpoint:
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-lite",
display_name: "Gemini 2.0 Flash Lite",
input_cost: 0.075,
output_cost: 0.30,
},
],
tokenizer: DiscourseAi::Tokenizer::GeminiTokenizer,
@ -69,11 +88,46 @@ module DiscourseAi
{
id: "open_ai",
models: [
{ name: "o3-mini", tokens: 200_000, display_name: "o3 Mini" },
{ name: "o1", tokens: 200_000, display_name: "o1" },
{ name: "gpt-4.1", tokens: 800_000, display_name: "GPT-4.1" },
{ name: "gpt-4.1-mini", tokens: 800_000, display_name: "GPT-4.1 Mini" },
{ name: "gpt-4.1-nano", tokens: 800_000, display_name: "GPT-4.1 Nano" },
{
name: "o3-mini",
tokens: 200_000,
display_name: "o3 Mini",
input_cost: 1.10,
cached_input_cost: 0.55,
output_cost: 4.40,
},
{
name: "o1",
tokens: 200_000,
display_name: "o1",
input_cost: 15,
cached_input_cost: 7.50,
output_cost: 60,
},
{
name: "gpt-4.1",
tokens: 800_000,
display_name: "GPT-4.1",
input_cost: 2,
cached_input_cost: 0.5,
output_cost: 8,
},
{
name: "gpt-4.1-mini",
tokens: 800_000,
display_name: "GPT-4.1 Mini",
input_cost: 0.40,
cached_input_cost: 0.10,
output_cost: 1.60,
},
{
name: "gpt-4.1-nano",
tokens: 800_000,
display_name: "GPT-4.1 Nano",
input_cost: 0.10,
cached_input_cost: 0.025,
output_cost: 0.40,
},
],
tokenizer: DiscourseAi::Tokenizer::OpenAiTokenizer,
endpoint: "https://api.openai.com/v1/chat/completions",
@ -86,11 +140,15 @@ module DiscourseAi
name: "Meta-Llama-3.3-70B-Instruct",
tokens: 131_072,
display_name: "Llama 3.3 70B",
input_cost: 0.60,
output_cost: 1.20,
},
{
name: "Meta-Llama-3.1-8B-Instruct",
tokens: 16_384,
display_name: "Llama 3.1 8B",
input_cost: 0.1,
output_cost: 0.20,
},
],
tokenizer: DiscourseAi::Tokenizer::Llama3Tokenizer,

74
lib/completions/report.rb
View File

@ -33,6 +33,27 @@ module DiscourseAi
stats.total_requests || 0
end
def total_spending
total = total_input_spending + total_output_spending + total_cached_input_spending
total.round(2)
end
def total_input_spending
model_costs.sum { |row| row.input_cost.to_f * row.total_request_tokens.to_i / 1_000_000.0 }
end
def total_output_spending
model_costs.sum do |row|
row.output_cost.to_f * row.total_response_tokens.to_i / 1_000_000.0
end
end
def total_cached_input_spending
model_costs.sum do |row|
row.cached_input_cost.to_f * row.total_cached_tokens.to_i / 1_000_000.0
end
end
def stats
@stats ||=
base_query.select(
@ -46,6 +67,24 @@ module DiscourseAi
]
end
def model_costs
@model_costs ||=
base_query
.joins("LEFT JOIN llm_models ON llm_models.name = language_model")
.group(
"llm_models.name, llm_models.input_cost, llm_models.output_cost, llm_models.cached_input_cost",
)
.select(
"llm_models.name",
"llm_models.input_cost",
"llm_models.output_cost",
"llm_models.cached_input_cost",
"SUM(COALESCE(request_tokens, 0)) as total_request_tokens",
"SUM(COALESCE(response_tokens, 0)) as total_response_tokens",
"SUM(COALESCE(cached_tokens, 0)) as total_cached_tokens",
)
end
def guess_period(period = nil)
period = nil if %i[day month hour].include?(period)
period ||
@ -76,7 +115,15 @@ module DiscourseAi
def user_breakdown
base_query
.joins(:user)
.group(:user_id, "users.username", "users.uploaded_avatar_id")
.joins("LEFT JOIN llm_models ON llm_models.name = language_model")
.group(
:user_id,
"users.username",
"users.uploaded_avatar_id",
"llm_models.input_cost",
"llm_models.output_cost",
"llm_models.cached_input_cost",
)
.order("usage_count DESC")
.limit(USER_LIMIT)
.select(
@ -87,12 +134,21 @@ module DiscourseAi
"SUM(COALESCE(cached_tokens,0)) as total_cached_tokens",
"SUM(COALESCE(request_tokens,0)) as total_request_tokens",
"SUM(COALESCE(response_tokens,0)) as total_response_tokens",
"SUM(COALESCE(request_tokens, 0)) * COALESCE(llm_models.input_cost, 0) / 1000000.0 as input_spending",
"SUM(COALESCE(response_tokens, 0)) * COALESCE(llm_models.output_cost, 0) / 1000000.0 as output_spending",
"SUM(COALESCE(cached_tokens, 0)) * COALESCE(llm_models.cached_input_cost, 0) / 1000000.0 as cached_input_spending",
)
end
def feature_breakdown
base_query
.group(:feature_name)
.joins("LEFT JOIN llm_models ON llm_models.name = language_model")
.group(
:feature_name,
"llm_models.input_cost",
"llm_models.output_cost",
"llm_models.cached_input_cost",
)
.order("usage_count DESC")
.select(
"case when coalesce(feature_name, '') = '' then '#{UNKNOWN_FEATURE}' else feature_name end as feature_name",
@ -101,12 +157,21 @@ module DiscourseAi
"SUM(COALESCE(cached_tokens,0)) as total_cached_tokens",
"SUM(COALESCE(request_tokens,0)) as total_request_tokens",
"SUM(COALESCE(response_tokens,0)) as total_response_tokens",
"SUM(COALESCE(request_tokens, 0)) * COALESCE(llm_models.input_cost, 0) / 1000000.0 as input_spending",
"SUM(COALESCE(response_tokens, 0)) * COALESCE(llm_models.output_cost, 0) / 1000000.0 as output_spending",
"SUM(COALESCE(cached_tokens, 0)) * COALESCE(llm_models.cached_input_cost, 0) / 1000000.0 as cached_input_spending",
)
end
def model_breakdown
base_query
.group(:language_model)
.joins("LEFT JOIN llm_models ON llm_models.name = language_model")
.group(
:language_model,
"llm_models.input_cost",
"llm_models.output_cost",
"llm_models.cached_input_cost",
)
.order("usage_count DESC")
.select(
"language_model as llm",
@ -115,6 +180,9 @@ module DiscourseAi
"SUM(COALESCE(cached_tokens,0)) as total_cached_tokens",
"SUM(COALESCE(request_tokens,0)) as total_request_tokens",
"SUM(COALESCE(response_tokens,0)) as total_response_tokens",
"SUM(COALESCE(request_tokens, 0)) * COALESCE(llm_models.input_cost, 0) / 1000000.0 as input_spending",
"SUM(COALESCE(response_tokens, 0)) * COALESCE(llm_models.output_cost, 0) / 1000000.0 as output_spending",
"SUM(COALESCE(cached_tokens, 0)) * COALESCE(llm_models.cached_input_cost, 0) / 1000000.0 as cached_input_spending",
)
end

3
spec/fabricators/llm_model_fabricator.rb
View File

@ -8,6 +8,9 @@ Fabricator(:llm_model) do
api_key "123"
url "https://api.openai.com/v1/chat/completions"
max_prompt_tokens 131_072
input_cost 10
cached_input_cost 2.5
output_cost 40
end
Fabricator(:anthropic_model, from: :llm_model) do

35
spec/requests/admin/ai_usage_controller_spec.rb
View File

@ -5,6 +5,7 @@ require "rails_helper"
RSpec.describe DiscourseAi::Admin::AiUsageController do
fab!(:admin)
fab!(:user)
fab!(:llm_model)
let(:usage_report_path) { "/admin/plugins/discourse-ai/ai-usage-report.json" }
before { SiteSetting.discourse_ai_enabled = true }
@ -35,6 +36,18 @@ RSpec.describe DiscourseAi::Admin::AiUsageController do
)
end
fab!(:log3) do
AiApiAuditLog.create!(
provider_id: 1,
feature_name: "ai_helper",
language_model: llm_model.name,
request_tokens: 300,
response_tokens: 150,
cached_tokens: 50,
created_at: 3.days.ago,
)
end
it "returns correct data structure" do
get usage_report_path
@ -55,7 +68,7 @@ RSpec.describe DiscourseAi::Admin::AiUsageController do
}
json = response.parsed_body
expect(json["summary"]["total_tokens"]).to eq(450) # sum of all tokens
expect(json["summary"]["total_tokens"]).to eq(900) # sum of all tokens
end
it "filters by feature" do
@ -79,6 +92,26 @@ RSpec.describe DiscourseAi::Admin::AiUsageController do
expect(models.first["total_tokens"]).to eq(300)
end
it "shows an estimated cost" do
get usage_report_path, params: { model: llm_model.name }
json = response.parsed_body
summary = json["summary"]
feature = json["features"].find { |f| f["feature_name"] == "ai_helper" }
expected_input_spending = llm_model.input_cost * log3.request_tokens / 1_000_000.0
expected_cached_input_spending =
llm_model.cached_input_cost * log3.cached_tokens / 1_000_000.0
expected_output_spending = llm_model.output_cost * log3.response_tokens / 1_000_000.0
expected_total_spending =
expected_input_spending + expected_cached_input_spending + expected_output_spending
expect(feature["input_spending"].to_s).to eq(expected_input_spending.to_s)
expect(feature["output_spending"].to_s).to eq(expected_output_spending.to_s)
expect(feature["cached_input_spending"].to_s).to eq(expected_cached_input_spending.to_s)
expect(summary["total_spending"].to_s).to eq(expected_total_spending.round(2).to_s)
end
it "handles different period groupings" do
get usage_report_path, params: { period: "hour" }
expect(response.status).to eq(200)