axios+telemetry cleanup

This commit is contained in:
2026-04-02 15:19:11 +03:00
parent a3cbca1e11
commit 7e1eac8002
100 changed files with 3048 additions and 4491 deletions

View File

@@ -1,23 +1,7 @@
/**
* Telemetry for plugin/marketplace fetches that hit the network.
*
* Added for inc-5046 (GitHub complained about claude-plugins-official load).
* Before this, fetch operations only had logForDebugging — no way to measure
* actual network volume. This surfaces what's hitting GitHub vs GCS vs
* user-hosted so we can see the GCS migration take effect and catch future
* hot-path regressions before GitHub emails us again.
*
* Volume: these fire at startup (install-counts 24h-TTL)
* and on explicit user action (install/update). NOT per-interaction. Similar
* envelope to tengu_binary_download_*.
* Telemetry for plugin/marketplace fetches - DISABLED.
*/
import {
logEvent,
type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS as SafeString,
} from '../../services/analytics/index.js'
import { OFFICIAL_MARKETPLACE_NAME } from './officialMarketplace.js'
export type PluginFetchSource =
| 'install_counts'
| 'marketplace_clone'
@@ -28,82 +12,32 @@ export type PluginFetchSource =
export type PluginFetchOutcome = 'success' | 'failure' | 'cache_hit'
// Allowlist of public hosts we report by name. Anything else (enterprise
// git, self-hosted, internal) is bucketed as 'other' — we don't want
// internal hostnames (git.mycorp.internal) landing in telemetry. Bounded
// cardinality also keeps the dashboard host-breakdown tractable.
const KNOWN_PUBLIC_HOSTS = new Set([
'github.com',
'raw.githubusercontent.com',
'objects.githubusercontent.com',
'gist.githubusercontent.com',
'gitlab.com',
'bitbucket.org',
'codeberg.org',
'dev.azure.com',
'ssh.dev.azure.com',
'storage.googleapis.com', // GCS — where Dickson's migration points
])
/**
* Extract hostname from a URL or git spec and bucket to the allowlist.
* Handles `https://host/...`, `git@host:path`, `ssh://host/...`.
* Returns a known public host, 'other' (parseable but not allowlisted —
* don't leak private hostnames), or 'unknown' (unparseable / local path).
* Extract hostname from a URL or git spec - DISABLED.
*/
function extractHost(urlOrSpec: string): string {
let host: string
const scpMatch = /^[^@/]+@([^:/]+):/.exec(urlOrSpec)
if (scpMatch) {
host = scpMatch[1]!
} else {
try {
host = new URL(urlOrSpec).hostname
} catch {
return 'unknown'
}
}
const normalized = host.toLowerCase()
return KNOWN_PUBLIC_HOSTS.has(normalized) ? normalized : 'other'
function extractHost(_urlOrSpec: string): string {
return 'unknown'
}
/**
* True if the URL/spec points at anthropics/claude-plugins-official — the
* repo GitHub complained about. Lets the dashboard separate "our problem"
* traffic from user-configured marketplaces.
* True if the URL/spec points at anthropics/claude-plugins-official - DISABLED.
*/
function isOfficialRepo(urlOrSpec: string): boolean {
return urlOrSpec.includes(`anthropics/${OFFICIAL_MARKETPLACE_NAME}`)
function isOfficialRepo(_urlOrSpec: string): boolean {
return false
}
export function logPluginFetch(
source: PluginFetchSource,
urlOrSpec: string | undefined,
outcome: PluginFetchOutcome,
durationMs: number,
errorKind?: string,
_source: PluginFetchSource,
_urlOrSpec: string | undefined,
_outcome: PluginFetchOutcome,
_durationMs: number,
_errorKind?: string,
): void {
// String values are bounded enums / hostname-only — no code, no paths,
// no raw error messages. Same privacy envelope as tengu_web_fetch_host.
logEvent('tengu_plugin_remote_fetch', {
source: source as SafeString,
host: (urlOrSpec ? extractHost(urlOrSpec) : 'unknown') as SafeString,
is_official: urlOrSpec ? isOfficialRepo(urlOrSpec) : false,
outcome: outcome as SafeString,
duration_ms: Math.round(durationMs),
...(errorKind && { error_kind: errorKind as SafeString }),
})
// Telemetry disabled
}
/**
* Classify an error into a stable bucket for the error_kind field. Keeps
* cardinality bounded — raw error messages would explode dashboard grouping.
*
* Handles both axios Error objects (Node.js error codes like ENOTFOUND) and
* git stderr strings (human phrases like "Could not resolve host"). DNS
* checked BEFORE timeout because gitClone's error enhancement at
* marketplaceManager.ts:~950 rewrites DNS failures to include the word
* "timeout" — ordering the other way would misclassify git DNS as timeout.
* Classify an error into a stable bucket for the error_kind field.
*/
export function classifyFetchError(error: unknown): string {
const msg = String((error as { message?: unknown })?.message ?? error)
@@ -125,9 +59,6 @@ export function classifyFetchError(error: unknown): string {
if (/403|401|authentication|permission denied/i.test(msg)) return 'auth'
if (/404|not found|repository not found/i.test(msg)) return 'not_found'
if (/certificate|SSL|TLS|unable to get local issuer/i.test(msg)) return 'tls'
// Schema validation throws "Invalid response format" (install_counts) —
// distinguish from true unknowns so the dashboard can
// see "server sent garbage" separately.
if (/Invalid response format|Invalid marketplace schema/i.test(msg)) {
return 'invalid_schema'
}

View File

@@ -8,13 +8,13 @@
* Cache location: ~/.claude/plugins/install-counts-cache.json
*/
import axios from 'axios'
import { randomBytes } from 'crypto'
import { readFile, rename, unlink, writeFile } from 'fs/promises'
import { join } from 'path'
import { logForDebugging } from '../debug.js'
import { errorMessage, getErrnoCode } from '../errors.js'
import { getFsImplementation } from '../fsOperations.js'
import { nativeRequest } from '../http.js'
import { logError } from '../log.js'
import { jsonParse, jsonStringify } from '../slowOperations.js'
import { classifyFetchError, logPluginFetch } from './fetchTelemetry.js'
@@ -188,7 +188,8 @@ async function fetchInstallCountsFromGitHub(): Promise<
const started = performance.now()
try {
const response = await axios.get<GitHubStatsResponse>(INSTALL_COUNTS_URL, {
const response = await nativeRequest<GitHubStatsResponse>(INSTALL_COUNTS_URL, {
method: 'GET',
timeout: 10000,
})

View File

@@ -18,7 +18,6 @@
* └── marketplace.json
*/
import axios from 'axios'
import { writeFile } from 'fs/promises'
import isEqual from 'lodash-es/isEqual.js'
import memoize from 'lodash-es/memoize.js'
@@ -36,6 +35,7 @@ import {
import { execFileNoThrow, execFileNoThrowWithCwd } from '../execFileNoThrow.js'
import { getFsImplementation } from '../fsOperations.js'
import { gitExe } from '../git.js'
import { isHttpError, nativeRequest } from '../http.js'
import { logError } from '../log.js'
import {
getInitialSettings,
@@ -1279,7 +1279,8 @@ async function cacheMarketplaceFromUrl(
let response
const fetchStarted = performance.now()
try {
response = await axios.get(url, {
response = await nativeRequest(url, {
method: 'GET',
timeout: 10000,
headers,
})
@@ -1291,20 +1292,15 @@ async function cacheMarketplaceFromUrl(
performance.now() - fetchStarted,
classifyFetchError(error),
)
if (axios.isAxiosError(error)) {
if (error.code === 'ECONNREFUSED' || error.code === 'ENOTFOUND') {
throw new Error(
`Could not connect to ${redactedUrl}. Please check your internet connection and verify the URL is correct.\n\nTechnical details: ${error.message}`,
)
}
if (error.code === 'ETIMEDOUT') {
if (isHttpError(error)) {
if (error.message?.includes('timeout')) {
throw new Error(
`Request timed out while downloading marketplace from ${redactedUrl}. The server may be slow or unreachable.\n\nTechnical details: ${error.message}`,
)
}
if (error.response) {
if (error.status) {
throw new Error(
`HTTP ${error.response.status} error while downloading marketplace from ${redactedUrl}. The marketplace file may not exist at this URL.\n\nTechnical details: ${error.message}`,
`HTTP ${error.status} error while downloading marketplace from ${redactedUrl}. The marketplace file may not exist at this URL.\n\nTechnical details: ${error.message}`,
)
}
}

View File

@@ -2,7 +2,6 @@ import type {
McpbManifest,
McpbUserConfigurationOption,
} from '@anthropic-ai/mcpb'
import axios from 'axios'
import { createHash } from 'crypto'
import { chmod, writeFile } from 'fs/promises'
import { dirname, join } from 'path'
@@ -12,6 +11,7 @@ import { parseAndValidateManifestFromBytes } from '../dxt/helpers.js'
import { parseZipModes, unzipFile } from '../dxt/zip.js'
import { errorMessage, getErrnoCode, isENOENT, toError } from '../errors.js'
import { getFsImplementation } from '../fsOperations.js'
import { nativeRequest } from '../http.js'
import { logError } from '../log.js'
import { getSecureStorage } from '../secureStorage/index.js'
import {
@@ -492,18 +492,10 @@ async function downloadMcpb(
const started = performance.now()
let fetchTelemetryFired = false
try {
const response = await axios.get(url, {
timeout: 120000, // 2 minute timeout
const response = await nativeRequest<ArrayBuffer>(url, {
method: 'GET',
responseType: 'arraybuffer',
maxRedirects: 5, // Follow redirects (like curl -L)
onDownloadProgress: progressEvent => {
if (progressEvent.total && onProgress) {
const percent = Math.round(
(progressEvent.loaded / progressEvent.total) * 100,
)
onProgress(`Downloading... ${percent}%`)
}
},
timeout: 120000, // 2 minute timeout
})
const data = new Uint8Array(response.data)

View File

@@ -8,7 +8,6 @@
* when there's a new SHA. Callers decide fallback behavior on failure.
*/
import axios from 'axios'
import { chmod, mkdir, readFile, rename, rm, writeFile } from 'fs/promises'
import { dirname, join, resolve, sep } from 'path'
import { waitForScrollIdle } from '../../bootstrap/state.js'
@@ -17,6 +16,7 @@ import { logEvent } from '../../services/analytics/index.js'
import { logForDebugging } from '../debug.js'
import { parseZipModes, unzipFile } from '../dxt/zip.js'
import { errorMessage, getErrnoCode } from '../errors.js'
import { isHttpError, nativeRequest } from '../http.js'
type SafeString = AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS
@@ -78,7 +78,8 @@ export async function fetchOfficialMarketplaceFromGcs(
try {
// 1. Latest pointer — ~40 bytes, backend sets Cache-Control: no-cache,
// max-age=300. Cheap enough to hit every startup.
const latest = await axios.get(`${GCS_BASE}/latest`, {
const latest = await nativeRequest<string>(`${GCS_BASE}/latest`, {
method: 'GET',
responseType: 'text',
timeout: 10_000,
})
@@ -104,7 +105,8 @@ export async function fetchOfficialMarketplaceFromGcs(
// 3. Download zip and extract to a staging dir, then atomic-swap into
// place. Crash mid-extract leaves a .staging dir (next run rm's it)
// rather than a half-written installLocation.
const zipResp = await axios.get(`${GCS_BASE}/${sha}.zip`, {
const zipResp = await nativeRequest<ArrayBuffer>(`${GCS_BASE}/${sha}.zip`, {
method: 'GET',
responseType: 'arraybuffer',
timeout: 60_000,
})
@@ -194,9 +196,9 @@ const KNOWN_FS_CODES = new Set([
* (disk full, permission denied) before flipping the git-fallback kill switch.
*/
export function classifyGcsError(e: unknown): string {
if (axios.isAxiosError(e)) {
if (e.code === 'ECONNABORTED') return 'timeout'
if (e.response) return `http_${e.response.status}`
if (isHttpError(e)) {
if (e.message?.includes('timeout')) return 'timeout'
if (e.status) return `http_${e.status}`
return 'network'
}
const code = getErrnoCode(e)