axios+telemetry cleanup
This commit is contained in:
@@ -1,23 +1,7 @@
|
||||
/**
|
||||
* Telemetry for plugin/marketplace fetches that hit the network.
|
||||
*
|
||||
* Added for inc-5046 (GitHub complained about claude-plugins-official load).
|
||||
* Before this, fetch operations only had logForDebugging — no way to measure
|
||||
* actual network volume. This surfaces what's hitting GitHub vs GCS vs
|
||||
* user-hosted so we can see the GCS migration take effect and catch future
|
||||
* hot-path regressions before GitHub emails us again.
|
||||
*
|
||||
* Volume: these fire at startup (install-counts 24h-TTL)
|
||||
* and on explicit user action (install/update). NOT per-interaction. Similar
|
||||
* envelope to tengu_binary_download_*.
|
||||
* Telemetry for plugin/marketplace fetches - DISABLED.
|
||||
*/
|
||||
|
||||
import {
|
||||
logEvent,
|
||||
type AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS as SafeString,
|
||||
} from '../../services/analytics/index.js'
|
||||
import { OFFICIAL_MARKETPLACE_NAME } from './officialMarketplace.js'
|
||||
|
||||
export type PluginFetchSource =
|
||||
| 'install_counts'
|
||||
| 'marketplace_clone'
|
||||
@@ -28,82 +12,32 @@ export type PluginFetchSource =
|
||||
|
||||
export type PluginFetchOutcome = 'success' | 'failure' | 'cache_hit'
|
||||
|
||||
// Allowlist of public hosts we report by name. Anything else (enterprise
|
||||
// git, self-hosted, internal) is bucketed as 'other' — we don't want
|
||||
// internal hostnames (git.mycorp.internal) landing in telemetry. Bounded
|
||||
// cardinality also keeps the dashboard host-breakdown tractable.
|
||||
const KNOWN_PUBLIC_HOSTS = new Set([
|
||||
'github.com',
|
||||
'raw.githubusercontent.com',
|
||||
'objects.githubusercontent.com',
|
||||
'gist.githubusercontent.com',
|
||||
'gitlab.com',
|
||||
'bitbucket.org',
|
||||
'codeberg.org',
|
||||
'dev.azure.com',
|
||||
'ssh.dev.azure.com',
|
||||
'storage.googleapis.com', // GCS — where Dickson's migration points
|
||||
])
|
||||
|
||||
/**
|
||||
* Extract hostname from a URL or git spec and bucket to the allowlist.
|
||||
* Handles `https://host/...`, `git@host:path`, `ssh://host/...`.
|
||||
* Returns a known public host, 'other' (parseable but not allowlisted —
|
||||
* don't leak private hostnames), or 'unknown' (unparseable / local path).
|
||||
* Extract hostname from a URL or git spec - DISABLED.
|
||||
*/
|
||||
function extractHost(urlOrSpec: string): string {
|
||||
let host: string
|
||||
const scpMatch = /^[^@/]+@([^:/]+):/.exec(urlOrSpec)
|
||||
if (scpMatch) {
|
||||
host = scpMatch[1]!
|
||||
} else {
|
||||
try {
|
||||
host = new URL(urlOrSpec).hostname
|
||||
} catch {
|
||||
return 'unknown'
|
||||
}
|
||||
}
|
||||
const normalized = host.toLowerCase()
|
||||
return KNOWN_PUBLIC_HOSTS.has(normalized) ? normalized : 'other'
|
||||
function extractHost(_urlOrSpec: string): string {
|
||||
return 'unknown'
|
||||
}
|
||||
|
||||
/**
|
||||
* True if the URL/spec points at anthropics/claude-plugins-official — the
|
||||
* repo GitHub complained about. Lets the dashboard separate "our problem"
|
||||
* traffic from user-configured marketplaces.
|
||||
* True if the URL/spec points at anthropics/claude-plugins-official - DISABLED.
|
||||
*/
|
||||
function isOfficialRepo(urlOrSpec: string): boolean {
|
||||
return urlOrSpec.includes(`anthropics/${OFFICIAL_MARKETPLACE_NAME}`)
|
||||
function isOfficialRepo(_urlOrSpec: string): boolean {
|
||||
return false
|
||||
}
|
||||
|
||||
export function logPluginFetch(
|
||||
source: PluginFetchSource,
|
||||
urlOrSpec: string | undefined,
|
||||
outcome: PluginFetchOutcome,
|
||||
durationMs: number,
|
||||
errorKind?: string,
|
||||
_source: PluginFetchSource,
|
||||
_urlOrSpec: string | undefined,
|
||||
_outcome: PluginFetchOutcome,
|
||||
_durationMs: number,
|
||||
_errorKind?: string,
|
||||
): void {
|
||||
// String values are bounded enums / hostname-only — no code, no paths,
|
||||
// no raw error messages. Same privacy envelope as tengu_web_fetch_host.
|
||||
logEvent('tengu_plugin_remote_fetch', {
|
||||
source: source as SafeString,
|
||||
host: (urlOrSpec ? extractHost(urlOrSpec) : 'unknown') as SafeString,
|
||||
is_official: urlOrSpec ? isOfficialRepo(urlOrSpec) : false,
|
||||
outcome: outcome as SafeString,
|
||||
duration_ms: Math.round(durationMs),
|
||||
...(errorKind && { error_kind: errorKind as SafeString }),
|
||||
})
|
||||
// Telemetry disabled
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify an error into a stable bucket for the error_kind field. Keeps
|
||||
* cardinality bounded — raw error messages would explode dashboard grouping.
|
||||
*
|
||||
* Handles both axios Error objects (Node.js error codes like ENOTFOUND) and
|
||||
* git stderr strings (human phrases like "Could not resolve host"). DNS
|
||||
* checked BEFORE timeout because gitClone's error enhancement at
|
||||
* marketplaceManager.ts:~950 rewrites DNS failures to include the word
|
||||
* "timeout" — ordering the other way would misclassify git DNS as timeout.
|
||||
* Classify an error into a stable bucket for the error_kind field.
|
||||
*/
|
||||
export function classifyFetchError(error: unknown): string {
|
||||
const msg = String((error as { message?: unknown })?.message ?? error)
|
||||
@@ -125,9 +59,6 @@ export function classifyFetchError(error: unknown): string {
|
||||
if (/403|401|authentication|permission denied/i.test(msg)) return 'auth'
|
||||
if (/404|not found|repository not found/i.test(msg)) return 'not_found'
|
||||
if (/certificate|SSL|TLS|unable to get local issuer/i.test(msg)) return 'tls'
|
||||
// Schema validation throws "Invalid response format" (install_counts) —
|
||||
// distinguish from true unknowns so the dashboard can
|
||||
// see "server sent garbage" separately.
|
||||
if (/Invalid response format|Invalid marketplace schema/i.test(msg)) {
|
||||
return 'invalid_schema'
|
||||
}
|
||||
|
||||
@@ -8,13 +8,13 @@
|
||||
* Cache location: ~/.claude/plugins/install-counts-cache.json
|
||||
*/
|
||||
|
||||
import axios from 'axios'
|
||||
import { randomBytes } from 'crypto'
|
||||
import { readFile, rename, unlink, writeFile } from 'fs/promises'
|
||||
import { join } from 'path'
|
||||
import { logForDebugging } from '../debug.js'
|
||||
import { errorMessage, getErrnoCode } from '../errors.js'
|
||||
import { getFsImplementation } from '../fsOperations.js'
|
||||
import { nativeRequest } from '../http.js'
|
||||
import { logError } from '../log.js'
|
||||
import { jsonParse, jsonStringify } from '../slowOperations.js'
|
||||
import { classifyFetchError, logPluginFetch } from './fetchTelemetry.js'
|
||||
@@ -188,7 +188,8 @@ async function fetchInstallCountsFromGitHub(): Promise<
|
||||
|
||||
const started = performance.now()
|
||||
try {
|
||||
const response = await axios.get<GitHubStatsResponse>(INSTALL_COUNTS_URL, {
|
||||
const response = await nativeRequest<GitHubStatsResponse>(INSTALL_COUNTS_URL, {
|
||||
method: 'GET',
|
||||
timeout: 10000,
|
||||
})
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
* └── marketplace.json
|
||||
*/
|
||||
|
||||
import axios from 'axios'
|
||||
import { writeFile } from 'fs/promises'
|
||||
import isEqual from 'lodash-es/isEqual.js'
|
||||
import memoize from 'lodash-es/memoize.js'
|
||||
@@ -36,6 +35,7 @@ import {
|
||||
import { execFileNoThrow, execFileNoThrowWithCwd } from '../execFileNoThrow.js'
|
||||
import { getFsImplementation } from '../fsOperations.js'
|
||||
import { gitExe } from '../git.js'
|
||||
import { isHttpError, nativeRequest } from '../http.js'
|
||||
import { logError } from '../log.js'
|
||||
import {
|
||||
getInitialSettings,
|
||||
@@ -1279,7 +1279,8 @@ async function cacheMarketplaceFromUrl(
|
||||
let response
|
||||
const fetchStarted = performance.now()
|
||||
try {
|
||||
response = await axios.get(url, {
|
||||
response = await nativeRequest(url, {
|
||||
method: 'GET',
|
||||
timeout: 10000,
|
||||
headers,
|
||||
})
|
||||
@@ -1291,20 +1292,15 @@ async function cacheMarketplaceFromUrl(
|
||||
performance.now() - fetchStarted,
|
||||
classifyFetchError(error),
|
||||
)
|
||||
if (axios.isAxiosError(error)) {
|
||||
if (error.code === 'ECONNREFUSED' || error.code === 'ENOTFOUND') {
|
||||
throw new Error(
|
||||
`Could not connect to ${redactedUrl}. Please check your internet connection and verify the URL is correct.\n\nTechnical details: ${error.message}`,
|
||||
)
|
||||
}
|
||||
if (error.code === 'ETIMEDOUT') {
|
||||
if (isHttpError(error)) {
|
||||
if (error.message?.includes('timeout')) {
|
||||
throw new Error(
|
||||
`Request timed out while downloading marketplace from ${redactedUrl}. The server may be slow or unreachable.\n\nTechnical details: ${error.message}`,
|
||||
)
|
||||
}
|
||||
if (error.response) {
|
||||
if (error.status) {
|
||||
throw new Error(
|
||||
`HTTP ${error.response.status} error while downloading marketplace from ${redactedUrl}. The marketplace file may not exist at this URL.\n\nTechnical details: ${error.message}`,
|
||||
`HTTP ${error.status} error while downloading marketplace from ${redactedUrl}. The marketplace file may not exist at this URL.\n\nTechnical details: ${error.message}`,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ import type {
|
||||
McpbManifest,
|
||||
McpbUserConfigurationOption,
|
||||
} from '@anthropic-ai/mcpb'
|
||||
import axios from 'axios'
|
||||
import { createHash } from 'crypto'
|
||||
import { chmod, writeFile } from 'fs/promises'
|
||||
import { dirname, join } from 'path'
|
||||
@@ -12,6 +11,7 @@ import { parseAndValidateManifestFromBytes } from '../dxt/helpers.js'
|
||||
import { parseZipModes, unzipFile } from '../dxt/zip.js'
|
||||
import { errorMessage, getErrnoCode, isENOENT, toError } from '../errors.js'
|
||||
import { getFsImplementation } from '../fsOperations.js'
|
||||
import { nativeRequest } from '../http.js'
|
||||
import { logError } from '../log.js'
|
||||
import { getSecureStorage } from '../secureStorage/index.js'
|
||||
import {
|
||||
@@ -492,18 +492,10 @@ async function downloadMcpb(
|
||||
const started = performance.now()
|
||||
let fetchTelemetryFired = false
|
||||
try {
|
||||
const response = await axios.get(url, {
|
||||
timeout: 120000, // 2 minute timeout
|
||||
const response = await nativeRequest<ArrayBuffer>(url, {
|
||||
method: 'GET',
|
||||
responseType: 'arraybuffer',
|
||||
maxRedirects: 5, // Follow redirects (like curl -L)
|
||||
onDownloadProgress: progressEvent => {
|
||||
if (progressEvent.total && onProgress) {
|
||||
const percent = Math.round(
|
||||
(progressEvent.loaded / progressEvent.total) * 100,
|
||||
)
|
||||
onProgress(`Downloading... ${percent}%`)
|
||||
}
|
||||
},
|
||||
timeout: 120000, // 2 minute timeout
|
||||
})
|
||||
|
||||
const data = new Uint8Array(response.data)
|
||||
|
||||
@@ -8,7 +8,6 @@
|
||||
* when there's a new SHA. Callers decide fallback behavior on failure.
|
||||
*/
|
||||
|
||||
import axios from 'axios'
|
||||
import { chmod, mkdir, readFile, rename, rm, writeFile } from 'fs/promises'
|
||||
import { dirname, join, resolve, sep } from 'path'
|
||||
import { waitForScrollIdle } from '../../bootstrap/state.js'
|
||||
@@ -17,6 +16,7 @@ import { logEvent } from '../../services/analytics/index.js'
|
||||
import { logForDebugging } from '../debug.js'
|
||||
import { parseZipModes, unzipFile } from '../dxt/zip.js'
|
||||
import { errorMessage, getErrnoCode } from '../errors.js'
|
||||
import { isHttpError, nativeRequest } from '../http.js'
|
||||
|
||||
type SafeString = AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS
|
||||
|
||||
@@ -78,7 +78,8 @@ export async function fetchOfficialMarketplaceFromGcs(
|
||||
try {
|
||||
// 1. Latest pointer — ~40 bytes, backend sets Cache-Control: no-cache,
|
||||
// max-age=300. Cheap enough to hit every startup.
|
||||
const latest = await axios.get(`${GCS_BASE}/latest`, {
|
||||
const latest = await nativeRequest<string>(`${GCS_BASE}/latest`, {
|
||||
method: 'GET',
|
||||
responseType: 'text',
|
||||
timeout: 10_000,
|
||||
})
|
||||
@@ -104,7 +105,8 @@ export async function fetchOfficialMarketplaceFromGcs(
|
||||
// 3. Download zip and extract to a staging dir, then atomic-swap into
|
||||
// place. Crash mid-extract leaves a .staging dir (next run rm's it)
|
||||
// rather than a half-written installLocation.
|
||||
const zipResp = await axios.get(`${GCS_BASE}/${sha}.zip`, {
|
||||
const zipResp = await nativeRequest<ArrayBuffer>(`${GCS_BASE}/${sha}.zip`, {
|
||||
method: 'GET',
|
||||
responseType: 'arraybuffer',
|
||||
timeout: 60_000,
|
||||
})
|
||||
@@ -194,9 +196,9 @@ const KNOWN_FS_CODES = new Set([
|
||||
* (disk full, permission denied) before flipping the git-fallback kill switch.
|
||||
*/
|
||||
export function classifyGcsError(e: unknown): string {
|
||||
if (axios.isAxiosError(e)) {
|
||||
if (e.code === 'ECONNABORTED') return 'timeout'
|
||||
if (e.response) return `http_${e.response.status}`
|
||||
if (isHttpError(e)) {
|
||||
if (e.message?.includes('timeout')) return 'timeout'
|
||||
if (e.status) return `http_${e.status}`
|
||||
return 'network'
|
||||
}
|
||||
const code = getErrnoCode(e)
|
||||
|
||||
Reference in New Issue
Block a user