discourse/discourse-ai
1
0
Fork 0
mirror of https://github.com/discourse/discourse-ai.git synced 2025-06-25 17:12:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

FEATURE: Add spending metrics to AI usage (#1268)

Browse Source 
This update adds metrics for estimated spending in AI usage. To make use of it, admins must add cost details to the LLM config page (input, output, and cached input costs per 1M tokens). After doing so, the metrics will appear in the AI usage dashboard as the AI plugin is used.
This commit is contained in:
Keegan George 2025-04-17 15:09:48 -07:00 committed by GitHub
parent e2b0287333
commit d26c7ac48d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 559 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
admin/assets/javascripts/discourse/routes/admin-plugins-show-discourse-ai-llms-edit.js
View File

@ -2,8 +2,17 @@ import DiscourseRoute from "discourse/routes/discourse";
export default class AdminPluginsShowDiscourseAiLlmsEdit extends DiscourseRoute { export default class AdminPluginsShowDiscourseAiLlmsEdit extends DiscourseRoute {
async model(params) { async model(params) {
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
const id = parseInt(params.id, 10); const id = parseInt(params.id, 10);
if (id < 0) {
// You shouldn't be able to access the edit page
// if the model is seeded
return this.router.transitionTo(
"adminPlugins.show.discourse-ai-llms.index"
);
}
const allLlms = this.modelFor("adminPlugins.show.discourse-ai-llms");
const record = allLlms.findBy("id", id); const record = allLlms.findBy("id", id);
record.provider_params = record.provider_params || {}; record.provider_params = record.provider_params || {};
return record; return record;

3
app/controllers/discourse_ai/admin/ai_llms_controller.rb
View File

@ -161,6 +161,9 @@ module DiscourseAi
:api_key, :api_key,
:enabled_chat_bot, :enabled_chat_bot,
:vision_enabled, :vision_enabled,
:input_cost,
:cached_input_cost,
:output_cost,
) )
provider = updating ? updating.provider : permitted[:provider] provider = updating ? updating.provider : permitted[:provider]

12
app/models/llm_model.rb
View File

@ -13,7 +13,14 @@ class LlmModel < ActiveRecord::Base
validates :url, presence: true, unless: -> { provider == BEDROCK_PROVIDER_NAME } validates :url, presence: true, unless: -> { provider == BEDROCK_PROVIDER_NAME }
validates_presence_of :name, :api_key validates_presence_of :name, :api_key
validates :max_prompt_tokens, numericality: { greater_than: 0 } validates :max_prompt_tokens, numericality: { greater_than: 0 }
validates :max_output_tokens, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true validates :input_cost,
:cached_input_cost,
:output_cost,
:max_output_tokens,
numericality: {
greater_than_or_equal_to: 0,
},
allow_nil: true
validate :required_provider_params validate :required_provider_params
scope :in_use, scope :in_use,
-> do -> do
@ -184,5 +191,8 @@ end
# enabled_chat_bot :boolean default(FALSE), not null # enabled_chat_bot :boolean default(FALSE), not null
# provider_params :jsonb # provider_params :jsonb
# vision_enabled :boolean default(FALSE), not null # vision_enabled :boolean default(FALSE), not null
# input_cost :float
# cached_input_cost :float
# output_cost :float
# max_output_tokens :integer # max_output_tokens :integer
# #

10
app/serializers/ai_usage_serializer.rb
View File

@ -22,6 +22,9 @@ class AiUsageSerializer < ApplicationSerializer
total_cached_tokens total_cached_tokens
total_request_tokens total_request_tokens
total_response_tokens total_response_tokens
input_spending
output_spending
cached_input_spending
], ],
) )
end end
@ -35,6 +38,9 @@ class AiUsageSerializer < ApplicationSerializer
total_cached_tokens total_cached_tokens
total_request_tokens total_request_tokens
total_response_tokens total_response_tokens
input_spending
output_spending
cached_input_spending
], ],
) )
end end
@ -49,6 +55,9 @@ class AiUsageSerializer < ApplicationSerializer
total_cached_tokens: user.total_cached_tokens, total_cached_tokens: user.total_cached_tokens,
total_request_tokens: user.total_request_tokens, total_request_tokens: user.total_request_tokens,
total_response_tokens: user.total_response_tokens, total_response_tokens: user.total_response_tokens,
input_spending: user.input_spending,
output_spending: user.output_spending,
cached_input_spending: user.cached_input_spending,
} }
end end
end end
@ -60,6 +69,7 @@ class AiUsageSerializer < ApplicationSerializer
total_request_tokens: object.total_request_tokens, total_request_tokens: object.total_request_tokens,
total_response_tokens: object.total_response_tokens, total_response_tokens: object.total_response_tokens,
total_requests: object.total_requests, total_requests: object.total_requests,
total_spending: object.total_spending,
date_range: { date_range: {
start: object.start_date, start: object.start_date,
end: object.end_date, end: object.end_date,

3
app/serializers/llm_model_serializer.rb
View File

@ -18,6 +18,9 @@ class LlmModelSerializer < ApplicationSerializer
:enabled_chat_bot, :enabled_chat_bot,
:provider_params, :provider_params,
:vision_enabled, :vision_enabled,
:input_cost,
:output_cost,
:cached_input_cost,
:used_by :used_by
has_one :user, serializer: BasicUserSerializer, embed: :object has_one :user, serializer: BasicUserSerializer, embed: :object

5
assets/javascripts/discourse/admin/models/ai-llm.js
View File

@ -15,7 +15,10 @@ export default class AiLlm extends RestModel {
"api_key", "api_key",
"enabled_chat_bot", "enabled_chat_bot",
"provider_params", "provider_params",
"vision_enabled" "vision_enabled",
"input_cost",
"cached_input_cost",
"output_cost"
); );
} }

512
assets/javascripts/discourse/components/ai-llm-editor-form.gjs
View File

@ -47,6 +47,9 @@ export default class AiLlmEditorForm extends Component {
name: modelInfo.name, name: modelInfo.name,
provider: info.provider, provider: info.provider,
provider_params: this.computeProviderParams(info.provider), provider_params: this.computeProviderParams(info.provider),
input_cost: modelInfo.input_cost,
output_cost: modelInfo.output_cost,
cached_input_cost: modelInfo.cached_input_cost,
}; };
} }
@ -63,6 +66,9 @@ export default class AiLlmEditorForm extends Component {
provider: model.provider, provider: model.provider,
enabled_chat_bot: model.enabled_chat_bot, enabled_chat_bot: model.enabled_chat_bot,
vision_enabled: model.vision_enabled, vision_enabled: model.vision_enabled,
input_cost: model.input_cost,
output_cost: model.output_cost,
cached_input_cost: model.cached_input_cost,
provider_params: this.computeProviderParams( provider_params: this.computeProviderParams(
model.provider, model.provider,
model.provider_params model.provider_params
@ -118,10 +124,6 @@ export default class AiLlmEditorForm extends Component {
return localized.join(", "); return localized.join(", ");
} }
get seeded() {
return this.args.model.id < 0;
}
get inUseWarning() { get inUseWarning() {
return i18n("discourse_ai.llms.in_use_warning", { return i18n("discourse_ai.llms.in_use_warning", {
settings: this.modulesUsingModel, settings: this.modulesUsingModel,
@ -271,15 +273,9 @@ export default class AiLlmEditorForm extends Component {
<Form <Form
@onSubmit={{this.save}} @onSubmit={{this.save}}
@data={{this.formData}} @data={{this.formData}}
class="ai-llm-editor {{if this.seeded 'seeded'}}" class="ai-llm-editor"
as |form data| as |form data|
> >
{{#if this.seeded}}
<form.Alert @icon="circle-info">
{{i18n "discourse_ai.llms.seeded_warning"}}
</form.Alert>
{{/if}}
{{#if this.modulesUsingModel}} {{#if this.modulesUsingModel}}
<form.Alert @icon="circle-info"> <form.Alert @icon="circle-info">
{{this.inUseWarning}} {{this.inUseWarning}}
@ -290,7 +286,6 @@ export default class AiLlmEditorForm extends Component {
@name="display_name" @name="display_name"
@title={{i18n "discourse_ai.llms.display_name"}} @title={{i18n "discourse_ai.llms.display_name"}}
@validation="required|length:1,100" @validation="required|length:1,100"
@disabled={{this.seeded}}
@format="large" @format="large"
@tooltip={{i18n "discourse_ai.llms.hints.display_name"}} @tooltip={{i18n "discourse_ai.llms.hints.display_name"}}
as |field| as |field|
@ -303,7 +298,6 @@ export default class AiLlmEditorForm extends Component {
@title={{i18n "discourse_ai.llms.name"}} @title={{i18n "discourse_ai.llms.name"}}
@tooltip={{i18n "discourse_ai.llms.hints.name"}} @tooltip={{i18n "discourse_ai.llms.hints.name"}}
@validation="required" @validation="required"
@disabled={{this.seeded}}
@format="large" @format="large"
as |field| as |field|
> >
@ -313,7 +307,6 @@ export default class AiLlmEditorForm extends Component {
<form.Field <form.Field
@name="provider" @name="provider"
@title={{i18n "discourse_ai.llms.provider"}} @title={{i18n "discourse_ai.llms.provider"}}
@disabled={{this.seeded}}
@format="large" @format="large"
@validation="required" @validation="required"
@onSet={{this.setProvider}} @onSet={{this.setProvider}}
@ -328,262 +321,289 @@ export default class AiLlmEditorForm extends Component {
</field.Select> </field.Select>
</form.Field> </form.Field>
{{#unless this.seeded}} {{#if (this.canEditURL data.provider)}}
{{#if (this.canEditURL data.provider)}}
<form.Field
@name="url"
@title={{i18n "discourse_ai.llms.url"}}
@validation="required"
@format="large"
as |field|
>
<field.Input />
</form.Field>
{{/if}}
<form.Field <form.Field
@name="api_key" @name="url"
@title={{i18n "discourse_ai.llms.api_key"}} @title={{i18n "discourse_ai.llms.url"}}
@validation="required" @validation="required"
@format="large" @format="large"
as |field| as |field|
> >
<field.Password /> <field.Input />
</form.Field> </form.Field>
{{/if}}
<form.Object @name="provider_params" as |object providerParamsData|> <form.Field
{{#each (this.providerParamsKeys providerParamsData) as |name|}} @name="api_key"
{{#let @title={{i18n "discourse_ai.llms.api_key"}}
(get (this.metaProviderParams data.provider) name) @validation="required"
as |params| @format="large"
}} as |field|
<object.Field >
@name={{name}} <field.Password />
@title={{i18n </form.Field>
(concat "discourse_ai.llms.provider_fields." name)
}}
@format="large"
as |field|
>
{{#if (eq params.type "enum")}}
<field.Select @includeNone={{false}} as |select|>
{{#each params.values as |option|}}
<select.Option
@value={{option.id}}
>{{option.name}}</select.Option>
{{/each}}
</field.Select>
{{else if (eq params.type "checkbox")}}
<field.Checkbox />
{{else}}
<field.Input @type={{params.type}} />
{{/if}}
</object.Field>
{{/let}}
{{/each}}
</form.Object>
<form.Field <form.Object @name="provider_params" as |object providerParamsData|>
@name="tokenizer" {{#each (this.providerParamsKeys providerParamsData) as |name|}}
@title={{i18n "discourse_ai.llms.tokenizer"}} {{#let
@disabled={{this.seeded}} (get (this.metaProviderParams data.provider) name)
@format="large" as |params|
@validation="required" }}
as |field| <object.Field
> @name={{name}}
<field.Select as |select|> @title={{i18n (concat "discourse_ai.llms.provider_fields." name)}}
{{#each this.tokenizers as |tokenizer|}} @format="large"
<select.Option as |field|
@value={{tokenizer.id}}
>{{tokenizer.name}}</select.Option>
{{/each}}
</field.Select>
</form.Field>
<form.Field
@name="max_prompt_tokens"
@title={{i18n "discourse_ai.llms.max_prompt_tokens"}}
@tooltip={{i18n "discourse_ai.llms.hints.max_prompt_tokens"}}
@validation="required"
@format="large"
as |field|
>
<field.Input @type="number" step="any" min="0" lang="en" />
</form.Field>
<form.Field
@name="max_output_tokens"
@title={{i18n "discourse_ai.llms.max_output_tokens"}}
@tooltip={{i18n "discourse_ai.llms.hints.max_output_tokens"}}
@format="large"
as |field|
>
<field.Input @type="number" step="any" min="0" lang="en" />
</form.Field>
<form.Field
@name="vision_enabled"
@title={{i18n "discourse_ai.llms.vision_enabled"}}
@tooltip={{i18n "discourse_ai.llms.hints.vision_enabled"}}
@format="large"
as |field|
>
<field.Checkbox />
</form.Field>
<form.Field
@name="enabled_chat_bot"
@title={{i18n "discourse_ai.llms.enabled_chat_bot"}}
@tooltip={{i18n "discourse_ai.llms.hints.enabled_chat_bot"}}
@format="large"
as |field|
>
<field.Checkbox />
</form.Field>
{{#if @model.user}}
<form.Container @title={{i18n "discourse_ai.llms.ai_bot_user"}}>
<a
class="avatar"
href={{@model.user.path}}
data-user-card={{@model.user.username}}
> >
{{Avatar @model.user.avatar_template "small"}} {{#if (eq params.type "enum")}}
</a> <field.Select @includeNone={{false}} as |select|>
<LinkTo @route="adminUser" @model={{this.adminUser}}> {{#each params.values as |option|}}
{{@model.user.username}} <select.Option
</LinkTo> @value={{option.id}}
</form.Container> >{{option.name}}</select.Option>
{{/if}} {{/each}}
</field.Select>
{{else if (eq params.type "checkbox")}}
<field.Checkbox />
{{else}}
<field.Input @type={{params.type}} />
{{/if}}
</object.Field>
{{/let}}
{{/each}}
</form.Object>
{{#if (gt data.llm_quotas.length 0)}} <form.Field
<form.Container @title={{i18n "discourse_ai.llms.quotas.title"}}> @name="tokenizer"
<table class="ai-llm-quotas__table"> @title={{i18n "discourse_ai.llms.tokenizer"}}
<thead class="ai-llm-quotas__table-head"> @format="large"
<tr class="ai-llm-quotas__header-row"> @validation="required"
<th class="ai-llm-quotas__header">{{i18n as |field|
"discourse_ai.llms.quotas.group" >
}}</th> <field.Select as |select|>
<th class="ai-llm-quotas__header">{{i18n {{#each this.tokenizers as |tokenizer|}}
"discourse_ai.llms.quotas.max_tokens" <select.Option
}}</th> @value={{tokenizer.id}}
<th class="ai-llm-quotas__header">{{i18n >{{tokenizer.name}}</select.Option>
"discourse_ai.llms.quotas.max_usages" {{/each}}
}}</th> </field.Select>
<th class="ai-llm-quotas__header">{{i18n </form.Field>
"discourse_ai.llms.quotas.duration"
}}</th> <form.Field
<th @name="max_prompt_tokens"
class="ai-llm-quotas__header ai-llm-quotas__header--actions" @title={{i18n "discourse_ai.llms.max_prompt_tokens"}}
></th> @tooltip={{i18n "discourse_ai.llms.hints.max_prompt_tokens"}}
<th></th> @validation="required"
</tr> @format="large"
</thead> as |field|
<tbody class="ai-llm-quotas__table-body"> >
<form.Collection <field.Input @type="number" step="any" min="0" lang="en" />
@name="llm_quotas" </form.Field>
@tagName="tr"
class="ai-llm-quotas__row" <form.InputGroup as |inputGroup|>
as |collection index collectionData| <inputGroup.Field
> @name="input_cost"
<td @title={{i18n "discourse_ai.llms.cost_input"}}
class="ai-llm-quotas__cell" @tooltip={{i18n "discourse_ai.llms.hints.cost_input"}}
>{{collectionData.group_name}}</td> @helpText={{i18n "discourse_ai.llms.hints.cost_measure"}}
<td class="ai-llm-quotas__cell"> as |field|
<collection.Field >
@name="max_tokens" <field.Input @type="number" step="any" min="0" lang="en" />
@title="max_tokens" </inputGroup.Field>
@showTitle={{false}}
as |field| <inputGroup.Field
> @name="cached_input_cost"
<field.Input @title={{i18n "discourse_ai.llms.cost_cached_input"}}
@type="number" @tooltip={{i18n "discourse_ai.llms.hints.cost_cached_input"}}
class="ai-llm-quotas__input" @helpText={{i18n "discourse_ai.llms.hints.cost_measure"}}
min="1" as |field|
/> >
</collection.Field> <field.Input @type="number" step="any" min="0" lang="en" />
</td> </inputGroup.Field>
<td class="ai-llm-quotas__cell">
<collection.Field <inputGroup.Field
@name="max_usages" @name="output_cost"
@title="max_usages" @title={{i18n "discourse_ai.llms.cost_output"}}
@showTitle={{false}} @tooltip={{i18n "discourse_ai.llms.hints.cost_output"}}
as |field| @helpText={{i18n "discourse_ai.llms.hints.cost_measure"}}
> as |field|
<field.Input >
@type="number" <field.Input @type="number" step="any" min="0" lang="en" />
class="ai-llm-quotas__input" </inputGroup.Field>
min="1" </form.InputGroup>
/>
</collection.Field> <form.Field
</td> @name="max_output_tokens"
<td class="ai-llm-quotas__cell"> @title={{i18n "discourse_ai.llms.max_output_tokens"}}
<collection.Field @tooltip={{i18n "discourse_ai.llms.hints.max_output_tokens"}}
@name="duration_seconds" @format="large"
@title="duration_seconds" as |field|
@showTitle={{false}} >
as |field| <field.Input @type="number" step="any" min="0" lang="en" />
> </form.Field>
<field.Custom>
<DurationSelector <form.Field
@value={{collectionData.duration_seconds}} @name="vision_enabled"
@onChange={{field.set}} @title={{i18n "discourse_ai.llms.vision_enabled"}}
/> @tooltip={{i18n "discourse_ai.llms.hints.vision_enabled"}}
</field.Custom> @format="large"
</collection.Field> as |field|
</td> >
<td> <field.Checkbox />
<form.Button </form.Field>
@icon="trash-can"
@action={{fn collection.remove index}} <form.Field
class="btn-danger ai-llm-quotas__delete-btn" @name="enabled_chat_bot"
@title={{i18n "discourse_ai.llms.enabled_chat_bot"}}
@tooltip={{i18n "discourse_ai.llms.hints.enabled_chat_bot"}}
@format="large"
as |field|
>
<field.Checkbox />
</form.Field>
{{#if @model.user}}
<form.Container @title={{i18n "discourse_ai.llms.ai_bot_user"}}>
<a
class="avatar"
href={{@model.user.path}}
data-user-card={{@model.user.username}}
>
{{Avatar @model.user.avatar_template "small"}}
</a>
<LinkTo @route="adminUser" @model={{this.adminUser}}>
{{@model.user.username}}
</LinkTo>
</form.Container>
{{/if}}
{{#if (gt data.llm_quotas.length 0)}}
<form.Container @title={{i18n "discourse_ai.llms.quotas.title"}}>
<table class="ai-llm-quotas__table">
<thead class="ai-llm-quotas__table-head">
<tr class="ai-llm-quotas__header-row">
<th class="ai-llm-quotas__header">{{i18n
"discourse_ai.llms.quotas.group"
}}</th>
<th class="ai-llm-quotas__header">{{i18n
"discourse_ai.llms.quotas.max_tokens"
}}</th>
<th class="ai-llm-quotas__header">{{i18n
"discourse_ai.llms.quotas.max_usages"
}}</th>
<th class="ai-llm-quotas__header">{{i18n
"discourse_ai.llms.quotas.duration"
}}</th>
<th
class="ai-llm-quotas__header ai-llm-quotas__header--actions"
></th>
<th></th>
</tr>
</thead>
<tbody class="ai-llm-quotas__table-body">
<form.Collection
@name="llm_quotas"
@tagName="tr"
class="ai-llm-quotas__row"
as |collection index collectionData|
>
<td
class="ai-llm-quotas__cell"
>{{collectionData.group_name}}</td>
<td class="ai-llm-quotas__cell">
<collection.Field
@name="max_tokens"
@title="max_tokens"
@showTitle={{false}}
as |field|
>
<field.Input
@type="number"
class="ai-llm-quotas__input"
min="1"
/> />
</td> </collection.Field>
</form.Collection> </td>
</tbody> <td class="ai-llm-quotas__cell">
</table> <collection.Field
</form.Container> @name="max_usages"
@title="max_usages"
@showTitle={{false}}
as |field|
>
<field.Input
@type="number"
class="ai-llm-quotas__input"
min="1"
/>
</collection.Field>
</td>
<td class="ai-llm-quotas__cell">
<collection.Field
@name="duration_seconds"
@title="duration_seconds"
@showTitle={{false}}
as |field|
>
<field.Custom>
<DurationSelector
@value={{collectionData.duration_seconds}}
@onChange={{field.set}}
/>
</field.Custom>
</collection.Field>
</td>
<td>
<form.Button
@icon="trash-can"
@action={{fn collection.remove index}}
class="btn-danger ai-llm-quotas__delete-btn"
/>
</td>
</form.Collection>
</tbody>
</table>
</form.Container>
<form.Button
@action={{fn
this.openAddQuotaModal
(fn form.addItemToCollection "llm_quotas")
}}
@icon="plus"
@label="discourse_ai.llms.quotas.add"
class="ai-llm-editor__add-quota-btn"
/>
{{/if}}
<form.Actions>
<form.Button
@action={{fn this.test data}}
@disabled={{this.testRunning}}
@label="discourse_ai.llms.tests.title"
/>
<form.Submit />
{{#if (eq data.llm_quotas.length 0)}}
<form.Button <form.Button
@action={{fn @action={{fn
this.openAddQuotaModal this.openAddQuotaModal
(fn form.addItemToCollection "llm_quotas") (fn form.addItemToCollection "llm_quotas")
}} }}
@icon="plus"
@label="discourse_ai.llms.quotas.add" @label="discourse_ai.llms.quotas.add"
class="ai-llm-editor__add-quota-btn" class="ai-llm-editor__add-quota-btn"
/> />
{{/if}} {{/if}}
<form.Actions> {{#unless @model.isNew}}
<form.Button <form.Button
@action={{fn this.test data}} @action={{this.delete}}
@disabled={{this.testRunning}} @label="discourse_ai.llms.delete"
@label="discourse_ai.llms.tests.title" class="btn-danger"
/> />
{{/unless}}
<form.Submit /> </form.Actions>
{{#if (eq data.llm_quotas.length 0)}}
<form.Button
@action={{fn
this.openAddQuotaModal
(fn form.addItemToCollection "llm_quotas")
}}
@label="discourse_ai.llms.quotas.add"
class="ai-llm-editor__add-quota-btn"
/>
{{/if}}
{{#unless @model.isNew}}
<form.Button
@action={{this.delete}}
@label="discourse_ai.llms.delete"
class="btn-danger"
/>
{{/unless}}
</form.Actions>
{{/unless}}
{{#if this.displayTestResult}} {{#if this.displayTestResult}}
<form.Container @format="full"> <form.Container @format="full">

61
assets/javascripts/discourse/components/ai-usage.gjs
View File

@ -2,6 +2,8 @@ import Component from "@glimmer/component";
import { tracked } from "@glimmer/tracking"; import { tracked } from "@glimmer/tracking";
import { fn, hash } from "@ember/helper"; import { fn, hash } from "@ember/helper";
import { action } from "@ember/object"; import { action } from "@ember/object";
import didInsert from "@ember/render-modifiers/modifiers/did-insert";
import didUpdate from "@ember/render-modifiers/modifiers/did-update";
import { LinkTo } from "@ember/routing"; import { LinkTo } from "@ember/routing";
import { service } from "@ember/service"; import { service } from "@ember/service";
import { eq, gt, lt } from "truth-helpers"; import { eq, gt, lt } from "truth-helpers";
@ -74,6 +76,22 @@ export default class AiUsage extends Component {
this.onFilterChange(); this.onFilterChange();
} }
@action
addCurrencyChar(element) {
element.querySelectorAll(".d-stat-tile__label").forEach((label) => {
if (
label.innerText.trim() === i18n("discourse_ai.usage.total_spending")
) {
const valueElement = label
.closest(".d-stat-tile")
?.querySelector(".d-stat-tile__value");
if (valueElement) {
valueElement.innerText = `$${valueElement.innerText}`;
}
}
});
}
@bind @bind
takeUsers(start, end) { takeUsers(start, end) {
return this.data.users.slice(start, end); return this.data.users.slice(start, end);
@ -153,6 +171,11 @@ export default class AiUsage extends Component {
value: this.data.summary.total_cached_tokens, value: this.data.summary.total_cached_tokens,
tooltip: i18n("discourse_ai.usage.stat_tooltips.cached_tokens"), tooltip: i18n("discourse_ai.usage.stat_tooltips.cached_tokens"),
}, },
{
label: i18n("discourse_ai.usage.total_spending"),
value: this.data.summary.total_spending,
tooltip: i18n("discourse_ai.usage.stat_tooltips.total_spending"),
},
]; ];
} }
@ -308,6 +331,11 @@ export default class AiUsage extends Component {
this.fetchData(); this.fetchData();
} }
totalSpending(inputSpending, cachedSpending, outputSpending) {
const total = inputSpending + cachedSpending + outputSpending;
return `$${total.toFixed(2)}`;
}
<template> <template>
<div class="ai-usage admin-detail"> <div class="ai-usage admin-detail">
<DPageSubheader <DPageSubheader
@ -376,9 +404,15 @@ export default class AiUsage extends Component {
class="ai-usage__summary" class="ai-usage__summary"
> >
<:content> <:content>
<DStatTiles as |tiles|> <DStatTiles
{{didInsert this.addCurrencyChar this.metrics}}
{{didUpdate this.addCurrencyChar this.metrics}}
as |tiles|
>
{{#each this.metrics as |metric|}} {{#each this.metrics as |metric|}}
<tiles.Tile <tiles.Tile
class="bar"
@label={{metric.label}} @label={{metric.label}}
@href={{metric.href}} @href={{metric.href}}
@value={{metric.value}} @value={{metric.value}}
@ -422,6 +456,7 @@ export default class AiUsage extends Component {
<th>{{i18n "discourse_ai.usage.feature"}}</th> <th>{{i18n "discourse_ai.usage.feature"}}</th>
<th>{{i18n "discourse_ai.usage.usage_count"}}</th> <th>{{i18n "discourse_ai.usage.usage_count"}}</th>
<th>{{i18n "discourse_ai.usage.total_tokens"}}</th> <th>{{i18n "discourse_ai.usage.total_tokens"}}</th>
<th>{{i18n "discourse_ai.usage.total_spending"}}</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
@ -438,6 +473,13 @@ export default class AiUsage extends Component {
class="ai-usage__features-cell" class="ai-usage__features-cell"
title={{feature.total_tokens}} title={{feature.total_tokens}}
>{{number feature.total_tokens}}</td> >{{number feature.total_tokens}}</td>
<td>
{{this.totalSpending
feature.input_spending
feature.cached_input_spending
feature.output_spending
}}
</td>
</tr> </tr>
{{/each}} {{/each}}
</tbody> </tbody>
@ -464,6 +506,8 @@ export default class AiUsage extends Component {
<th>{{i18n "discourse_ai.usage.model"}}</th> <th>{{i18n "discourse_ai.usage.model"}}</th>
<th>{{i18n "discourse_ai.usage.usage_count"}}</th> <th>{{i18n "discourse_ai.usage.usage_count"}}</th>
<th>{{i18n "discourse_ai.usage.total_tokens"}}</th> <th>{{i18n "discourse_ai.usage.total_tokens"}}</th>
<th>{{i18n "discourse_ai.usage.total_spending"}}</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
@ -478,6 +522,13 @@ export default class AiUsage extends Component {
class="ai-usage__models-cell" class="ai-usage__models-cell"
title={{model.total_tokens}} title={{model.total_tokens}}
>{{number model.total_tokens}}</td> >{{number model.total_tokens}}</td>
<td>
{{this.totalSpending
model.input_spending
model.cached_input_spending
model.output_spending
}}
</td>
</tr> </tr>
{{/each}} {{/each}}
</tbody> </tbody>
@ -511,6 +562,7 @@ export default class AiUsage extends Component {
}}</th> }}</th>
<th>{{i18n "discourse_ai.usage.usage_count"}}</th> <th>{{i18n "discourse_ai.usage.usage_count"}}</th>
<th>{{i18n "discourse_ai.usage.total_tokens"}}</th> <th>{{i18n "discourse_ai.usage.total_tokens"}}</th>
<th>{{i18n "discourse_ai.usage.total_spending"}}</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
@ -535,6 +587,13 @@ export default class AiUsage extends Component {
class="ai-usage__users-cell" class="ai-usage__users-cell"
title={{user.total_tokens}} title={{user.total_tokens}}
>{{number user.total_tokens}}</td> >{{number user.total_tokens}}</td>
<td>
{{this.totalSpending
user.input_spending
user.cached_input_spending
user.output_spending
}}
</td>
</tr> </tr>
{{/each}} {{/each}}
</tbody> </tbody>

10
config/locales/client.en.yml
View File

@ -239,6 +239,7 @@ en:
net_request_tokens: "Net request tokens" net_request_tokens: "Net request tokens"
cached_tokens: "Cached tokens" cached_tokens: "Cached tokens"
cached_request_tokens: "Cached request tokens" cached_request_tokens: "Cached request tokens"
total_spending: "Estimated cost"
no_users: "No user usage data found" no_users: "No user usage data found"
no_models: "No model usage data found" no_models: "No model usage data found"
no_features: "No feature usage data found" no_features: "No feature usage data found"
@ -249,6 +250,7 @@ en:
request_tokens: "Tokens used when the LLM tries to understand what you are saying" request_tokens: "Tokens used when the LLM tries to understand what you are saying"
response_tokens: "Tokens used when the LLM responds to your prompt" response_tokens: "Tokens used when the LLM responds to your prompt"
cached_tokens: "Previously processed request tokens that the LLM reuses to optimize performance and cost" cached_tokens: "Previously processed request tokens that the LLM reuses to optimize performance and cost"
total_spending: "Cumulative cost of all tokens used by the LLMs based on specified cost metrics added to LLM configuration settings"
periods: periods:
last_day: "Last 24 hours" last_day: "Last 24 hours"
last_week: "Last week" last_week: "Last week"
@ -404,6 +406,10 @@ en:
enabled_chat_bot: "Allow AI bot selector" enabled_chat_bot: "Allow AI bot selector"
vision_enabled: "Vision enabled" vision_enabled: "Vision enabled"
ai_bot_user: "AI bot User" ai_bot_user: "AI bot User"
cost_input: "Input cost"
cost_cached_input: "Cached input cost"
cost_output: "Output cost"
save: "Save" save: "Save"
edit: "Edit" edit: "Edit"
saved: "LLM model saved" saved: "LLM model saved"
@ -487,6 +493,10 @@ en:
name: "We include this in the API call to specify which model we'll use" name: "We include this in the API call to specify which model we'll use"
vision_enabled: "If enabled, the AI will attempt to understand images. It depends on the model being used supporting vision. Supported by latest models from Anthropic, Google, and OpenAI." vision_enabled: "If enabled, the AI will attempt to understand images. It depends on the model being used supporting vision. Supported by latest models from Anthropic, Google, and OpenAI."
enabled_chat_bot: "If enabled, users can select this model when creating PMs with the AI bot" enabled_chat_bot: "If enabled, users can select this model when creating PMs with the AI bot"
cost_input: "The input cost per 1M tokens for this model"
cost_cached_input: "The cached input cost per 1M tokens for this model"
cost_output: "The output cost per 1M tokens for this model"
cost_measure: "$/1M tokens"
providers: providers:
aws_bedrock: "AWS Bedrock" aws_bedrock: "AWS Bedrock"
anthropic: "Anthropic" anthropic: "Anthropic"

9
db/migrate/20250416215039_add_cost_metrics_to_llm_model.rb Normal file
View File

@ -0,0 +1,9 @@
# frozen_string_literal: true
class AddCostMetricsToLlmModel < ActiveRecord::Migration[7.2]
def change
add_column :llm_models, :input_cost, :float
add_column :llm_models, :cached_input_cost, :float
add_column :llm_models, :output_cost, :float
end
end

72
lib/completions/llm.rb
View File

@ -30,9 +30,26 @@ module DiscourseAi
name: "claude-3-7-sonnet", name: "claude-3-7-sonnet",
tokens: 200_000, tokens: 200_000,
display_name: "Claude 3.7 Sonnet", display_name: "Claude 3.7 Sonnet",
input_cost: 3,
cached_input_cost: 0.30,
output_cost: 15,
},
{
name: "claude-3-5-haiku",
tokens: 200_000,
display_name: "Claude 3.5 Haiku",
input_cost: 0.80,
cached_input_cost: 0.08,
output_cost: 4,
},
{
name: "claude-3-opus",
tokens: 200_000,
display_name: "Claude 3 Opus",
input_cost: 15,
cached_input_cost: 1.50,
output_cost: 75,
}, },
{ name: "claude-3-5-haiku", tokens: 200_000, display_name: "Claude 3.5 Haiku" },
{ name: "claude-3-opus", tokens: 200_000, display_name: "Claude 3 Opus" },
], ],
tokenizer: DiscourseAi::Tokenizer::AnthropicTokenizer, tokenizer: DiscourseAi::Tokenizer::AnthropicTokenizer,
endpoint: "https://api.anthropic.com/v1/messages", endpoint: "https://api.anthropic.com/v1/messages",
@ -61,6 +78,8 @@ module DiscourseAi
endpoint: endpoint:
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-lite", "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-lite",
display_name: "Gemini 2.0 Flash Lite", display_name: "Gemini 2.0 Flash Lite",
input_cost: 0.075,
output_cost: 0.30,
}, },
], ],
tokenizer: DiscourseAi::Tokenizer::GeminiTokenizer, tokenizer: DiscourseAi::Tokenizer::GeminiTokenizer,
@ -69,11 +88,46 @@ module DiscourseAi
{ {
id: "open_ai", id: "open_ai",
models: [ models: [
{ name: "o3-mini", tokens: 200_000, display_name: "o3 Mini" }, {
{ name: "o1", tokens: 200_000, display_name: "o1" }, name: "o3-mini",
{ name: "gpt-4.1", tokens: 800_000, display_name: "GPT-4.1" }, tokens: 200_000,
{ name: "gpt-4.1-mini", tokens: 800_000, display_name: "GPT-4.1 Mini" }, display_name: "o3 Mini",
{ name: "gpt-4.1-nano", tokens: 800_000, display_name: "GPT-4.1 Nano" }, input_cost: 1.10,
cached_input_cost: 0.55,
output_cost: 4.40,
},
{
name: "o1",
tokens: 200_000,
display_name: "o1",
input_cost: 15,
cached_input_cost: 7.50,
output_cost: 60,
},
{
name: "gpt-4.1",
tokens: 800_000,
display_name: "GPT-4.1",
input_cost: 2,
cached_input_cost: 0.5,
output_cost: 8,
},
{
name: "gpt-4.1-mini",
tokens: 800_000,
display_name: "GPT-4.1 Mini",
input_cost: 0.40,
cached_input_cost: 0.10,
output_cost: 1.60,
},
{
name: "gpt-4.1-nano",
tokens: 800_000,
display_name: "GPT-4.1 Nano",
input_cost: 0.10,
cached_input_cost: 0.025,
output_cost: 0.40,
},
], ],
tokenizer: DiscourseAi::Tokenizer::OpenAiTokenizer, tokenizer: DiscourseAi::Tokenizer::OpenAiTokenizer,
endpoint: "https://api.openai.com/v1/chat/completions", endpoint: "https://api.openai.com/v1/chat/completions",
@ -86,11 +140,15 @@ module DiscourseAi
name: "Meta-Llama-3.3-70B-Instruct", name: "Meta-Llama-3.3-70B-Instruct",
tokens: 131_072, tokens: 131_072,
display_name: "Llama 3.3 70B", display_name: "Llama 3.3 70B",
input_cost: 0.60,
output_cost: 1.20,
}, },
{ {
name: "Meta-Llama-3.1-8B-Instruct", name: "Meta-Llama-3.1-8B-Instruct",
tokens: 16_384, tokens: 16_384,
display_name: "Llama 3.1 8B", display_name: "Llama 3.1 8B",
input_cost: 0.1,
output_cost: 0.20,
}, },
], ],
tokenizer: DiscourseAi::Tokenizer::Llama3Tokenizer, tokenizer: DiscourseAi::Tokenizer::Llama3Tokenizer,

74
lib/completions/report.rb
View File

@ -33,6 +33,27 @@ module DiscourseAi
stats.total_requests || 0 stats.total_requests || 0
end end
def total_spending
total = total_input_spending + total_output_spending + total_cached_input_spending
total.round(2)
end
def total_input_spending
model_costs.sum { |row| row.input_cost.to_f * row.total_request_tokens.to_i / 1_000_000.0 }
end
def total_output_spending
model_costs.sum do |row|
row.output_cost.to_f * row.total_response_tokens.to_i / 1_000_000.0
end
end
def total_cached_input_spending
model_costs.sum do |row|
row.cached_input_cost.to_f * row.total_cached_tokens.to_i / 1_000_000.0
end
end
def stats def stats
@stats ||= @stats ||=
base_query.select( base_query.select(
@ -46,6 +67,24 @@ module DiscourseAi
] ]
end end
def model_costs
@model_costs ||=
base_query
.joins("LEFT JOIN llm_models ON llm_models.name = language_model")
.group(
"llm_models.name, llm_models.input_cost, llm_models.output_cost, llm_models.cached_input_cost",
)
.select(
"llm_models.name",
"llm_models.input_cost",
"llm_models.output_cost",
"llm_models.cached_input_cost",
"SUM(COALESCE(request_tokens, 0)) as total_request_tokens",
"SUM(COALESCE(response_tokens, 0)) as total_response_tokens",
"SUM(COALESCE(cached_tokens, 0)) as total_cached_tokens",
)
end
def guess_period(period = nil) def guess_period(period = nil)
period = nil if %i[day month hour].include?(period) period = nil if %i[day month hour].include?(period)
period || period ||
@ -76,7 +115,15 @@ module DiscourseAi
def user_breakdown def user_breakdown
base_query base_query
.joins(:user) .joins(:user)
.group(:user_id, "users.username", "users.uploaded_avatar_id") .joins("LEFT JOIN llm_models ON llm_models.name = language_model")
.group(
:user_id,
"users.username",
"users.uploaded_avatar_id",
"llm_models.input_cost",
"llm_models.output_cost",
"llm_models.cached_input_cost",
)
.order("usage_count DESC") .order("usage_count DESC")
.limit(USER_LIMIT) .limit(USER_LIMIT)
.select( .select(
@ -87,12 +134,21 @@ module DiscourseAi
"SUM(COALESCE(cached_tokens,0)) as total_cached_tokens", "SUM(COALESCE(cached_tokens,0)) as total_cached_tokens",
"SUM(COALESCE(request_tokens,0)) as total_request_tokens", "SUM(COALESCE(request_tokens,0)) as total_request_tokens",
"SUM(COALESCE(response_tokens,0)) as total_response_tokens", "SUM(COALESCE(response_tokens,0)) as total_response_tokens",
"SUM(COALESCE(request_tokens, 0)) * COALESCE(llm_models.input_cost, 0) / 1000000.0 as input_spending",
"SUM(COALESCE(response_tokens, 0)) * COALESCE(llm_models.output_cost, 0) / 1000000.0 as output_spending",
"SUM(COALESCE(cached_tokens, 0)) * COALESCE(llm_models.cached_input_cost, 0) / 1000000.0 as cached_input_spending",
) )
end end
def feature_breakdown def feature_breakdown
base_query base_query
.group(:feature_name) .joins("LEFT JOIN llm_models ON llm_models.name = language_model")
.group(
:feature_name,
"llm_models.input_cost",
"llm_models.output_cost",
"llm_models.cached_input_cost",
)
.order("usage_count DESC") .order("usage_count DESC")
.select( .select(
"case when coalesce(feature_name, '') = '' then '#{UNKNOWN_FEATURE}' else feature_name end as feature_name", "case when coalesce(feature_name, '') = '' then '#{UNKNOWN_FEATURE}' else feature_name end as feature_name",
@ -101,12 +157,21 @@ module DiscourseAi
"SUM(COALESCE(cached_tokens,0)) as total_cached_tokens", "SUM(COALESCE(cached_tokens,0)) as total_cached_tokens",
"SUM(COALESCE(request_tokens,0)) as total_request_tokens", "SUM(COALESCE(request_tokens,0)) as total_request_tokens",
"SUM(COALESCE(response_tokens,0)) as total_response_tokens", "SUM(COALESCE(response_tokens,0)) as total_response_tokens",
"SUM(COALESCE(request_tokens, 0)) * COALESCE(llm_models.input_cost, 0) / 1000000.0 as input_spending",
"SUM(COALESCE(response_tokens, 0)) * COALESCE(llm_models.output_cost, 0) / 1000000.0 as output_spending",
"SUM(COALESCE(cached_tokens, 0)) * COALESCE(llm_models.cached_input_cost, 0) / 1000000.0 as cached_input_spending",
) )
end end
def model_breakdown def model_breakdown
base_query base_query
.group(:language_model) .joins("LEFT JOIN llm_models ON llm_models.name = language_model")
.group(
:language_model,
"llm_models.input_cost",
"llm_models.output_cost",
"llm_models.cached_input_cost",
)
.order("usage_count DESC") .order("usage_count DESC")
.select( .select(
"language_model as llm", "language_model as llm",
@ -115,6 +180,9 @@ module DiscourseAi
"SUM(COALESCE(cached_tokens,0)) as total_cached_tokens", "SUM(COALESCE(cached_tokens,0)) as total_cached_tokens",
"SUM(COALESCE(request_tokens,0)) as total_request_tokens", "SUM(COALESCE(request_tokens,0)) as total_request_tokens",
"SUM(COALESCE(response_tokens,0)) as total_response_tokens", "SUM(COALESCE(response_tokens,0)) as total_response_tokens",
"SUM(COALESCE(request_tokens, 0)) * COALESCE(llm_models.input_cost, 0) / 1000000.0 as input_spending",
"SUM(COALESCE(response_tokens, 0)) * COALESCE(llm_models.output_cost, 0) / 1000000.0 as output_spending",
"SUM(COALESCE(cached_tokens, 0)) * COALESCE(llm_models.cached_input_cost, 0) / 1000000.0 as cached_input_spending",
) )
end end

3
spec/fabricators/llm_model_fabricator.rb
View File

@ -8,6 +8,9 @@ Fabricator(:llm_model) do
api_key "123" api_key "123"
url "https://api.openai.com/v1/chat/completions" url "https://api.openai.com/v1/chat/completions"
max_prompt_tokens 131_072 max_prompt_tokens 131_072
input_cost 10
cached_input_cost 2.5
output_cost 40
end end
Fabricator(:anthropic_model, from: :llm_model) do Fabricator(:anthropic_model, from: :llm_model) do

35
spec/requests/admin/ai_usage_controller_spec.rb
View File

@ -5,6 +5,7 @@ require "rails_helper"
RSpec.describe DiscourseAi::Admin::AiUsageController do RSpec.describe DiscourseAi::Admin::AiUsageController do
fab!(:admin) fab!(:admin)
fab!(:user) fab!(:user)
fab!(:llm_model)
let(:usage_report_path) { "/admin/plugins/discourse-ai/ai-usage-report.json" } let(:usage_report_path) { "/admin/plugins/discourse-ai/ai-usage-report.json" }
before { SiteSetting.discourse_ai_enabled = true } before { SiteSetting.discourse_ai_enabled = true }
@ -35,6 +36,18 @@ RSpec.describe DiscourseAi::Admin::AiUsageController do
) )
end end
fab!(:log3) do
AiApiAuditLog.create!(
provider_id: 1,
feature_name: "ai_helper",
language_model: llm_model.name,
request_tokens: 300,
response_tokens: 150,
cached_tokens: 50,
created_at: 3.days.ago,
)
end
it "returns correct data structure" do it "returns correct data structure" do
get usage_report_path get usage_report_path
@ -55,7 +68,7 @@ RSpec.describe DiscourseAi::Admin::AiUsageController do
} }
json = response.parsed_body json = response.parsed_body
expect(json["summary"]["total_tokens"]).to eq(450) # sum of all tokens expect(json["summary"]["total_tokens"]).to eq(900) # sum of all tokens
end end
it "filters by feature" do it "filters by feature" do
@ -79,6 +92,26 @@ RSpec.describe DiscourseAi::Admin::AiUsageController do
expect(models.first["total_tokens"]).to eq(300) expect(models.first["total_tokens"]).to eq(300)
end end
it "shows an estimated cost" do
get usage_report_path, params: { model: llm_model.name }
json = response.parsed_body
summary = json["summary"]
feature = json["features"].find { |f| f["feature_name"] == "ai_helper" }
expected_input_spending = llm_model.input_cost * log3.request_tokens / 1_000_000.0
expected_cached_input_spending =
llm_model.cached_input_cost * log3.cached_tokens / 1_000_000.0
expected_output_spending = llm_model.output_cost * log3.response_tokens / 1_000_000.0
expected_total_spending =
expected_input_spending + expected_cached_input_spending + expected_output_spending
expect(feature["input_spending"].to_s).to eq(expected_input_spending.to_s)
expect(feature["output_spending"].to_s).to eq(expected_output_spending.to_s)
expect(feature["cached_input_spending"].to_s).to eq(expected_cached_input_spending.to_s)
expect(summary["total_spending"].to_s).to eq(expected_total_spending.round(2).to_s)
end
it "handles different period groupings" do it "handles different period groupings" do
get usage_report_path, params: { period: "hour" } get usage_report_path, params: { period: "hour" }
expect(response.status).to eq(200) expect(response.status).to eq(200)