diff --git a/package-lock.json b/package-lock.json index b333948d2..f522f7ba7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -69,6 +69,7 @@ "packages/malloy-render", "packages/malloy-render-validator", "packages/malloy-syntax-highlight", + "packages/malloy-language-server", "test", "profiler" ] @@ -6446,6 +6447,10 @@ "resolved": "packages/malloy-interfaces", "link": true }, + "node_modules/@malloydata/malloy-language-server": { + "resolved": "packages/malloy-language-server", + "link": true + }, "node_modules/@malloydata/malloy-profiler": { "resolved": "profiler", "link": true @@ -28761,6 +28766,58 @@ "node": ">=0.10.0" } }, + "node_modules/vscode-jsonrpc": { + "version": "8.2.1", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.1.tgz", + "integrity": "sha512-kdjOSJ2lLIn7r1rtrMbbNCHjyMPfRnowdKjBQ+mGq6NAW5QY2bEZC/khaC5OR8svbbjvLEaIXkOq45e2X9BIbQ==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/vscode-languageserver": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/vscode-languageserver/-/vscode-languageserver-9.0.1.tgz", + "integrity": "sha512-woByF3PDpkHFUreUa7Hos7+pUWdeWMXRd26+ZX2A8cFx6v/JPTtd4/uN0/jB6XQHYaOlHbio03NTHCqrgG5n7g==", + "license": "MIT", + "dependencies": { + "vscode-languageserver-protocol": "3.17.5" + }, + "bin": { + "installServerIntoExtension": "bin/installServerIntoExtension" + } + }, + "node_modules/vscode-languageserver-protocol": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-protocol/-/vscode-languageserver-protocol-3.17.5.tgz", + "integrity": "sha512-mb1bvRJN8SVznADSGWM9u/b07H7Ecg0I3OgXDuLdn307rl/J3A9YD6/eYOssqhecL27hK1IPZAsaqh00i/Jljg==", + "license": "MIT", + "dependencies": { + "vscode-jsonrpc": "8.2.0", + "vscode-languageserver-types": "3.17.5" + } + }, + "node_modules/vscode-languageserver-protocol/node_modules/vscode-jsonrpc": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/vscode-jsonrpc/-/vscode-jsonrpc-8.2.0.tgz", + "integrity": "sha512-C+r0eKJUIfiDIfwJhria30+TYWPtuHJXHtI7J0YlOmKAo7ogxP20T0zxB7HZQIFhIyvoBPwWskjxrvAtfjyZfA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/vscode-languageserver-textdocument": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.12.tgz", + "integrity": "sha512-cxWNPesCnQCcMPeenjKKsOCKQZ/L6Tv19DTRIGuLWe32lyzWhihGVJ/rcckZXJxfdKCFvRLS3fpBIsV/ZGX4zA==", + "license": "MIT" + }, + "node_modules/vscode-languageserver-types": { + "version": "3.17.5", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.17.5.tgz", + "integrity": "sha512-Ld1VelNuX9pdF39h2Hgaeb5hEZM2Z3jUrrMgWQAu82jMtZp7p3vJT3BzToKtZI7NgQssZje5o0zryOrhQvzQAg==", + "license": "MIT" + }, "node_modules/vscode-oniguruma": { "version": "1.7.0", "dev": true, @@ -28775,7 +28832,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.1.0.tgz", "integrity": "sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ==", - "dev": true, "license": "MIT" }, "node_modules/w3c-xmlserializer": { @@ -30393,6 +30449,32 @@ "url": "https://github.com/sponsors/isaacs" } }, + "packages/malloy-language-server": { + "name": "@malloydata/malloy-language-server", + "version": "0.0.396", + "license": "MIT", + "dependencies": { + "@malloydata/malloy": "0.0.396", + "@malloydata/malloy-interfaces": "0.0.396", + "@malloydata/malloy-sql": "0.0.396", + "@malloydata/render-validator": "0.0.396", + "lodash": "^4.17.21", + "vscode-jsonrpc": "^8.1.0", + "vscode-languageserver": "^9.0.1", + "vscode-languageserver-textdocument": "^1.0.8", + "vscode-uri": "^3.0.7" + }, + "bin": { + "malloy-language-server": "bin/malloy-language-server.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "@malloydata/db-publisher": "0.0.396", + "@malloydata/malloy-connections": "0.0.396" + } + }, "packages/malloy-malloy-sql": { "name": "@malloydata/malloy-sql", "version": "0.0.396", diff --git a/package.json b/package.json index 272046577..1f89a84c6 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "packages/malloy-render", "packages/malloy-render-validator", "packages/malloy-syntax-highlight", + "packages/malloy-language-server", "test", "profiler" ] diff --git a/packages/malloy-language-server/bin/malloy-language-server.js b/packages/malloy-language-server/bin/malloy-language-server.js new file mode 100755 index 000000000..2b099cbca --- /dev/null +++ b/packages/malloy-language-server/bin/malloy-language-server.js @@ -0,0 +1,79 @@ +#!/usr/bin/env node + +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +const args = process.argv.slice(2); + +if (args[0] === 'check') { + // One-shot diagnostic mode + const files = []; + let format = 'text'; + let globalConfigDir; + + for (let i = 1; i < args.length; i++) { + if (args[i] === '--format' && args[i + 1]) { + format = args[i + 1]; + i++; + } else if (args[i] === '--global-config-dir' && args[i + 1]) { + globalConfigDir = args[i + 1]; + i++; + } else if (!args[i].startsWith('-')) { + files.push(args[i]); + } + } + + if (files.length === 0) { + console.error('Usage: malloy-language-server check [file2.malloy ...] [--format text|json] [--global-config-dir ]'); + process.exit(1); + } + + const {check} = require('../dist/check'); + check(files, {format, globalConfigDir}).then((exitCode) => { + process.exit(exitCode); + }).catch((err) => { + console.error(err); + process.exit(1); + }); +} else { + // Long-lived LSP server mode (stdio) + const {createConnection, ProposedFeatures} = require('vscode-languageserver/node'); + const {createServer, CommonConnectionManager, NodeURLReader} = require('../dist/index'); + const os = require('os'); + const {pathToFileURL} = require('url'); + + // Try to load backends (optional) + try { require('@malloydata/malloy-connections'); } catch {} + try { require('@malloydata/db-publisher'); } catch {} + + const {ConnectionFactory} = require('../dist/common/connections/types'); + + const connection = createConnection(ProposedFeatures.all); + const urlReader = new NodeURLReader(); + const connectionManager = new CommonConnectionManager( + {}, // Empty factory — backends registered via side-effect imports above + { + expandHome: (path) => path.replace(/^~/, os.homedir()), + pathToFileURL: (path) => pathToFileURL(path), + } + ); + connectionManager.setURLReader(urlReader); + + // Parse optional flags + let globalConfigDir; + for (let i = 0; i < args.length; i++) { + if (args[i] === '--global-config-dir' && args[i + 1]) { + globalConfigDir = args[i + 1]; + i++; + } + } + if (globalConfigDir) { + connectionManager.setGlobalConfigDirectory(globalConfigDir); + } else if (process.env['MALLOY_GLOBAL_CONFIG_DIR']) { + connectionManager.setGlobalConfigDirectory(process.env['MALLOY_GLOBAL_CONFIG_DIR']); + } + + createServer(connection, connectionManager, {urlReader}); +} diff --git a/packages/malloy-language-server/claude-code-plugin/.claude-plugin/plugin.json b/packages/malloy-language-server/claude-code-plugin/.claude-plugin/plugin.json new file mode 100644 index 000000000..0426b8608 --- /dev/null +++ b/packages/malloy-language-server/claude-code-plugin/.claude-plugin/plugin.json @@ -0,0 +1,8 @@ +{ + "name": "malloy-lsp", + "version": "1.0.0", + "description": "Malloy language intelligence for Claude Code — diagnostics, completions, hover, go-to-definition for .malloy files", + "author": {"name": "Malloy Contributors"}, + "repository": "https://github.com/malloydata/malloy", + "keywords": ["malloy", "lsp", "data", "sql"] +} diff --git a/packages/malloy-language-server/claude-code-plugin/.lsp.json b/packages/malloy-language-server/claude-code-plugin/.lsp.json new file mode 100644 index 000000000..df36e89f2 --- /dev/null +++ b/packages/malloy-language-server/claude-code-plugin/.lsp.json @@ -0,0 +1,10 @@ +{ + "malloy": { + "command": "malloy-language-server", + "extensionToLanguage": { + ".malloy": "malloy", + ".malloysql": "malloy-sql", + ".malloynb": "malloy-notebook" + } + } +} diff --git a/packages/malloy-language-server/package.json b/packages/malloy-language-server/package.json new file mode 100644 index 000000000..b72109999 --- /dev/null +++ b/packages/malloy-language-server/package.json @@ -0,0 +1,55 @@ +{ + "name": "@malloydata/malloy-language-server", + "version": "0.0.396", + "license": "MIT", + "description": "Malloy Language Server Protocol implementation — diagnostics, completions, hover, go-to-definition for .malloy files", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "malloy-language-server": "./bin/malloy-language-server.js" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "./node": { + "types": "./dist/node/index.d.ts", + "default": "./dist/node/index.js" + }, + "./package.json": "./package.json" + }, + "typesVersions": { + "*": { + "index": ["./dist/index.d.ts"], + "node": ["./dist/node/index.d.ts"] + } + }, + "engines": { + "node": ">=20" + }, + "homepage": "https://github.com/malloydata/malloy#readme", + "repository": { + "type": "git", + "url": "https://github.com/malloydata/malloy" + }, + "scripts": { + "build": "tsc --build", + "clean": "tsc --build --clean && rm -f tsconfig.tsbuildinfo" + }, + "dependencies": { + "@malloydata/malloy": "0.0.396", + "@malloydata/malloy-sql": "0.0.396", + "@malloydata/malloy-interfaces": "0.0.396", + "@malloydata/render-validator": "0.0.396", + "vscode-languageserver": "^9.0.1", + "vscode-languageserver-textdocument": "^1.0.8", + "vscode-jsonrpc": "^8.1.0", + "vscode-uri": "^3.0.7", + "lodash": "^4.17.21" + }, + "optionalDependencies": { + "@malloydata/malloy-connections": "0.0.396", + "@malloydata/db-publisher": "0.0.396" + } +} diff --git a/packages/malloy-language-server/src/check.ts b/packages/malloy-language-server/src/check.ts new file mode 100644 index 000000000..8bb2f2489 --- /dev/null +++ b/packages/malloy-language-server/src/check.ts @@ -0,0 +1,182 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import {pathToFileURL} from 'url'; +import {TextDocument} from 'vscode-languageserver-textdocument'; +import type {Diagnostic} from 'vscode-languageserver'; +import {DiagnosticSeverity} from 'vscode-languageserver'; +import type {TextDocuments} from 'vscode-languageserver'; +import {getMalloyDiagnostics} from './diagnostics'; +import type {TranslateCacheLogger} from './translate_cache'; +import {TranslateCache} from './translate_cache'; +import {CommonConnectionManager} from './common/connection_manager'; +import {NodeURLReader} from './node_url_reader'; + +function severityLabel(severity: DiagnosticSeverity | undefined): string { + switch (severity) { + case DiagnosticSeverity.Error: + return 'error'; + case DiagnosticSeverity.Warning: + return 'warning'; + case DiagnosticSeverity.Information: + return 'info'; + case DiagnosticSeverity.Hint: + return 'hint'; + default: + return 'error'; + } +} + +interface CheckOptions { + format?: 'text' | 'json'; + globalConfigDir?: string; +} + +export async function check( + files: string[], + options: CheckOptions = {} +): Promise { + const {format = 'text', globalConfigDir} = options; + const urlReader = new NodeURLReader(); + + // Dynamically try to load backends; they're optional dependencies + try { + require('@malloydata/malloy-connections'); + } catch { + // Not installed — only config-file connections will work + } + try { + require('@malloydata/db-publisher'); + } catch { + // Not installed + } + + const connectionManager = new CommonConnectionManager( + {}, // Empty ConnectionFactory — backends registered via side-effect imports above + { + expandHome: (p: string) => + p.replace(/^~/, process.env['HOME'] || '/root'), + pathToFileURL: (p: string) => pathToFileURL(p), + } + ); + connectionManager.setURLReader(urlReader); + + if (globalConfigDir) { + connectionManager.setGlobalConfigDirectory(globalConfigDir); + } else if (process.env['MALLOY_GLOBAL_CONFIG_DIR']) { + connectionManager.setGlobalConfigDirectory( + process.env['MALLOY_GLOBAL_CONFIG_DIR'] + ); + } + + const logger: TranslateCacheLogger = { + info: () => {}, + debug: () => {}, + error: (msg: string) => console.error(msg), + }; + + // Create a minimal TextDocuments mock for TranslateCache + const documentsMap = new Map(); + const documents = { + get: (uri: string) => documentsMap.get(uri), + all: () => Array.from(documentsMap.values()), + } as unknown as TextDocuments; + + const translateCache = new TranslateCache( + documents, + logger, + connectionManager, + urlReader + ); + + let hasErrors = false; + const allDiagnostics: { + file: string; + diagnostics: {[uri: string]: Diagnostic[]}; + }[] = []; + + for (const file of files) { + const absolutePath = path.resolve(file); + const fileUrl = pathToFileURL(absolutePath); + const uri = fileUrl.toString(); + + let content: string; + try { + content = await fs.promises.readFile(absolutePath, 'utf-8'); + } catch (err) { + console.error(`Error reading file: ${file}: ${err}`); + hasErrors = true; + continue; + } + + // Determine language ID from extension + const ext = path.extname(file).toLowerCase(); + let languageId = 'malloy'; + if (ext === '.malloysql') { + languageId = 'malloy-sql'; + } else if (ext === '.malloynb') { + languageId = 'malloy-notebook'; + } + + const document = TextDocument.create(uri, languageId, 0, content); + documentsMap.set(uri, document); + + // Set workspace root to file's directory + connectionManager.setWorkspaceRoots([new URL('./', fileUrl)]); + + try { + const diagnostics = await getMalloyDiagnostics(translateCache, document); + + allDiagnostics.push({file, diagnostics}); + + for (const [diagUri, diags] of Object.entries(diagnostics)) { + for (const diag of diags) { + if (diag.severity === DiagnosticSeverity.Error) { + hasErrors = true; + } + if (format === 'text') { + const displayUri = + diagUri === uri + ? file + : diagUri.startsWith('file://') + ? diagUri.replace('file://', '') + : diagUri; + const line = diag.range.start.line + 1; + const col = diag.range.start.character + 1; + const sev = severityLabel(diag.severity); + console.log( + `${displayUri}:${line}:${col}: ${sev}: ${diag.message}` + ); + } + } + } + } catch (err) { + hasErrors = true; + if (format === 'text') { + console.error(`${file}: ${err}`); + } + } + } + + if (format === 'json') { + const output = allDiagnostics.map(({file, diagnostics}) => ({ + file, + diagnostics: Object.entries(diagnostics).flatMap(([uri, diags]) => + diags.map(d => ({ + uri, + severity: severityLabel(d.severity), + range: d.range, + message: d.message, + code: d.code, + })) + ), + })); + console.log(JSON.stringify(output, null, 2)); + } + + return hasErrors ? 1 : 0; +} diff --git a/packages/malloy-language-server/src/code_actions/code_actions.ts b/packages/malloy-language-server/src/code_actions/code_actions.ts new file mode 100644 index 000000000..46276b5d2 --- /dev/null +++ b/packages/malloy-language-server/src/code_actions/code_actions.ts @@ -0,0 +1,63 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type {CodeAction, Command, Range} from 'vscode-languageserver'; +import {CodeActionKind} from 'vscode-languageserver'; +import type {TextDocument} from 'vscode-languageserver-textdocument'; +import type {TranslateCache} from '../translate_cache'; +import type {LogMessage} from '@malloydata/malloy'; +import {MalloyError} from '@malloydata/malloy'; + +export async function getMalloyCodeAction( + translateCache: TranslateCache, + document: TextDocument, + range: Range +): Promise<(CodeAction | Command)[] | null> { + const problems: LogMessage[] = []; + try { + const model = await translateCache.translateWithCache( + document.uri, + document.languageId + ); + if (model?.problems) { + problems.push(...model.problems); + } + } catch (error) { + if (error instanceof MalloyError) { + problems.push(...error.problems); + } + } + const actions: CodeAction[] = []; + for (const problem of problems) { + if (problem.at?.range) { + const par = problem.at.range; + if ( + par.start.line === range.start.line && + par.start.character === range.start.character && + par.end.line === range.end.line && + par.end.character === range.end.character && + problem.replacement + ) { + const edit = { + changes: { + [document.uri]: [ + { + range, + newText: problem.replacement, + }, + ], + }, + }; + const codeAction: CodeAction = { + title: `Replace with ${problem.replacement}`, + kind: CodeActionKind.QuickFix, + edit, + }; + actions.push(codeAction); + } + } + } + return actions; +} diff --git a/packages/malloy-language-server/src/common/completion_docs.ts b/packages/malloy-language-server/src/common/completion_docs.ts new file mode 100644 index 000000000..89e4eb82c --- /dev/null +++ b/packages/malloy-language-server/src/common/completion_docs.ts @@ -0,0 +1,332 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +const DOCS_ROOT = 'https://docs.malloydata.dev/documentation'; + +const MODEL_SOURCE_DOC = `Use \`source\` to name, describe, and augment a data source. + +\`\`\`malloy +source: flights is duckdb.table('flights.parquet') extend { + measure: flight_count is count() +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/source). +`; + +const MODEL_QUERY_DOC = `Use \`query\` to define a query which can be run within this document. + +\`\`\`malloy +query: flights_by_carrier is flights -> { + group_by: carrier + aggregate: flight_count +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/query). +`; + +const MODEL_RUN_DOC = `Use \`run\` to define an anonymous query which can be run within this document. + +\`\`\`malloy +run: flights -> { + group_by: carrier + aggregate: flight_count +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/statement#run-statements). +`; + +const QUERY_GROUP_BY_DOC = `Use the \`group_by\` clause to specify dimensions by which to group aggregate calculations. + +\`\`\`malloy +run: flights -> { + group_by: carrier + aggregate: flight_count +} +\`\`\` +`; + +const QUERY_ORDER_BY_DOC = `Use \`order_by\` to control the ordering of results. + +\`\`\`malloy +run: flights -> { + group_by: carrier + order_by: carrier asc +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/order_by#explicit-ordering). +`; + +const QUERY_SELECT_DOC = `Use \`select\` to retrieve dimensional values without grouping or aggregating. + +\`\`\`malloy +run: flights -> { + select: id2, carrier, dep_time + limit: 10 +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/user_guides/basic#select). +`; + +const QUERY_INDEX_DOC = `Use \`index\` to produce a search index. + +\`\`\`malloy +run: flights -> { + index: * on flight_count +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}). +`; + +const QUERY_AGGREGATE_DOC = `Use \`aggregate\` to perform aggregate computations like \`count()\` or \`sum()\`. + +\`\`\`malloy +run: flights -> { + group_by: carrier + aggregate: + flight_count is count() + total_distance is sum(distance) +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/aggregates). +`; + +const QUERY_CALCULATE_DOC = `Use \`calculate\` to perform aggregate computations like \`count()\` or \`sum()\`. + +\`\`\`malloy +query: flights -> { + group_by: carrier + calculate: previous_carrier is lag(carrier) +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/calculations). +`; + +const QUERY_TOP_DOC = `Use \`top\` to restrict the number of results returned. + +\`\`\`malloy +run: flights -> { + top: 10 + group_by: carrier + aggregate: flight_count +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/order_by#limiting). +`; + +const QUERY_LIMIT_DOC = `Use \`limit\` to restrict the number of results returned. + +\`\`\`malloy +run: flights -> { + select: * + limit: 10 +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/order_by#limiting). +`; + +const QUERY_WHERE_DOC = `Use \`where\` to narrow down results. + +\`\`\`malloy +run: flights -> { + where: origin.state = 'CA' + aggregate: flight_count +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/filters). +`; + +const QUERY_HAVING_DOC = `Use \`having\` to narrow down results based on conditions of aggregate values. + +\`\`\`malloy +run: flights -> { + group_by: carrier + aggregate: total_distance + having: total_distance > 1000 +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/filters). +`; + +const QUERY_NEST_DOC = `Use \`nest\` to include a nested view. + +\`\`\`malloy +run: flights -> { + group_by: carrier + nest: by_origin_state is { + group_by: origin.state + aggregate: flight_count + } +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/nesting). +`; + +const SOURCE_DIMENSION_DOC = `Use \`dimension\` to define a non-aggregate calculation. + +\`\`\`malloy +source: flights is duckdb.table('flights.parquet') extend { + dimension: distance_km is distance * 1.609 +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/fields#dimensions). +`; + +const SOURCE_MEASURE_DOC = `Use \`measure\` to define an aggregate calculation. + +\`\`\`malloy +source: flights is duckdb.table('flights.parquet') extend { + measure: flight_count is count() +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/fields#measures). +`; + +const SOURCE_VIEW_DOC = `Use \`view\` to define a named view which can be referenced and/or refined in queries. + +\`\`\`malloy +source: flights is duckdb.table('flights.parquet') extend { + view: by_carrier is { + group_by: carrier, + aggregate: flight_count + } +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/fields#queries). +`; + +const SOURCE_JOIN_ONE_DOC = `Use \`join_one\` to define a joined explore which has one row for each row in the source table. + +\`\`\`malloy +source: flights is duckdb.table('flights.parquet') extend { + join_one: carriers with carrier + join_one: origin is airports with origin_code +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/join). +`; + +const SOURCE_JOIN_MANY_DOC = `Use \`join_many\` to define a joined explore which has many rows for each row in the source table. + +\`\`\`malloy +source: users is table('users') extend { + join_many: orders is table('orders') on id = orders.user_id and orders.user_id != null +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/join). +`; + +const SOURCE_JOIN_CROSS_DOC = `Use \`join_cross\` to define a join via a cross product, resulting in many rows on each side of the join. + +View [the full documentation](${DOCS_ROOT}/language/join). +`; + +const SOURCE_WHERE_DOC = `Use \`where\` to limit the limit the rows of an explore. + +\`\`\`malloy +source: long_flights is flights extend { + where: distance > 1000 +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/filters). +`; + +const SOURCE_PRIMARY_KEY_DOC = `Use \`primary_key\` to specify a primary key for joining. + +\`\`\`malloy +source: flights is duckdb.table('flights.parquet') extend { + primary_key: id2 +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/explore#primary-keys). +`; + +const SOURCE_RENAME_DOC = `Use \`rename\` to rename a field from the source explore/table. + +\`\`\`malloy +source: flights is duckdb.table('flights.parquet') extend { + rename: origin_code is origin +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/explore#renaming-fields). +`; + +const SOURCE_ACCEPT_DOC = `Use \`accept\` to specify which fields to include from the source explore/table. + +\`\`\`malloy +source: airports is table('malloy-data.faa.airports') extend { + accept: [ id, name, code, city, state, elevation ] +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/explore#limiting-access-to-fields). +`; + +const SOURCE_EXCEPT_DOC = `Use \`except\` to specify which fields to exclude from the source explore/table. + +\`\`\`malloy +source: airports is table('malloy-data.faa.airports') extend { + except: [ c_ldg_rts, aero_cht, cntl_twr ] +} +\`\`\` + +View [the full documentation](${DOCS_ROOT}/language/explore#limiting-access-to-fields). +`; + +export const COMPLETION_DOCS: { + [kind: string]: {[property: string]: string}; +} = { + model_property: { + source: MODEL_SOURCE_DOC, + query: MODEL_QUERY_DOC, + run: MODEL_RUN_DOC, + }, + query_property: { + group_by: QUERY_GROUP_BY_DOC, + order_by: QUERY_ORDER_BY_DOC, + select: QUERY_SELECT_DOC, + index: QUERY_INDEX_DOC, + aggregate: QUERY_AGGREGATE_DOC, + top: QUERY_TOP_DOC, + limit: QUERY_LIMIT_DOC, + where: QUERY_WHERE_DOC, + having: QUERY_HAVING_DOC, + nest: QUERY_NEST_DOC, + calculate: QUERY_CALCULATE_DOC, + }, + explore_property: { + dimension: SOURCE_DIMENSION_DOC, + measure: SOURCE_MEASURE_DOC, + view: SOURCE_VIEW_DOC, + join_one: SOURCE_JOIN_ONE_DOC, + join_many: SOURCE_JOIN_MANY_DOC, + join_cross: SOURCE_JOIN_CROSS_DOC, + where: SOURCE_WHERE_DOC, + primary_key: SOURCE_PRIMARY_KEY_DOC, + rename: SOURCE_RENAME_DOC, + accept: SOURCE_ACCEPT_DOC, + except: SOURCE_EXCEPT_DOC, + }, +}; diff --git a/packages/malloy-language-server/src/common/connection_manager.ts b/packages/malloy-language-server/src/common/connection_manager.ts new file mode 100644 index 000000000..c5bfec4be --- /dev/null +++ b/packages/malloy-language-server/src/common/connection_manager.ts @@ -0,0 +1,402 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import { + MalloyConfig, + discoverConfig, + contextOverlay, + getRegisteredConnectionTypes, +} from '@malloydata/malloy'; +import type { + ConfigOverlays, + Overlay, + Connection, + ConnectionConfigEntry, + LookupConnection, + URLReader, +} from '@malloydata/malloy'; +import type {UnresolvedConnectionConfigEntry} from './types/connection_manager_types'; +import type {ConnectionFactory} from './connections/types'; + +export type SecretResolver = (key: string) => Promise; + +const SECRET_KEY_PATTERN = /^connections\.[^.]+\.[A-Za-z_][A-Za-z0-9_]*$/; + +export function isSecretKeyReference( + value: unknown +): value is {secretKey: string} { + return ( + typeof value === 'object' && + value !== null && + 'secretKey' in value && + typeof (value as Record)['secretKey'] === 'string' + ); +} + +function translateSettingsEntries( + settings: Record +): Record { + const out: Record = {}; + for (const [name, entry] of Object.entries(settings)) { + const translated: ConnectionConfigEntry = {is: entry.is}; + for (const [k, v] of Object.entries(entry)) { + if (k === 'is') continue; + if (isSecretKeyReference(v)) { + translated[k] = {secret: v.secretKey}; + } else { + translated[k] = v; + } + } + out[name] = translated; + } + return out; +} + +export interface HostAdapter { + expandHome?(path: string): string; + pathToFileURL?(path: string): URL; +} + +interface CachedConfig { + config: MalloyConfig; + source: 'discovered' | 'global' | 'defaults'; + configURL?: URL; +} + +function isAncestor(ancestor: URL, descendant: URL): boolean { + const a = ancestor.toString(); + const d = descendant.toString(); + return d === a || d.startsWith(a.endsWith('/') ? a : a + '/'); +} + +export class CommonConnectionManager { + currentRowLimit = 50; + private workspaceRoots: URL[] = []; + private globalConfigDirectory = ''; + private secretResolver: SecretResolver | undefined; + private urlReader: URLReader | undefined; + private hostAdapter: HostAdapter; + + private settingsConfig: + | Record + | undefined; + + private configCache = new Map(); + private directoryIndex = new Map(); + + constructor( + private connectionFactory: ConnectionFactory, + hostAdapter?: HostAdapter + ) { + this.hostAdapter = hostAdapter ?? {}; + } + + public setURLReader(reader: URLReader): void { + this.urlReader = reader; + this.invalidateCache(); + } + + public getURLReader(): URLReader | undefined { + return this.urlReader; + } + + public setGlobalConfigDirectory(dir: string): void { + if (this.globalConfigDirectory !== dir) { + this.globalConfigDirectory = dir; + this.invalidateCache(); + } + } + + public setSecretResolver(resolver: SecretResolver): void { + this.secretResolver = resolver; + } + + public setWorkspaceRoots(roots: (URL | string)[]): void { + this.workspaceRoots = roots.map(r => + typeof r === 'string' ? new URL(r.endsWith('/') ? r : r + '/') : r + ); + this.invalidateCache(); + } + + public setCurrentRowLimit(rowLimit: number): void { + this.currentRowLimit = rowLimit; + } + + public getCurrentRowLimit(): number | undefined { + return this.currentRowLimit; + } + + public getNewFormatConfig(): + | Record + | undefined { + return this.settingsConfig; + } + + public setConnectionsConfig( + connectionsConfig: Record + ): void { + this.settingsConfig = connectionsConfig; + this.invalidateCache(); + } + + public notifyConfigFileChanged(): void { + this.invalidateCache(); + } + + public async getConnectionLookup( + fileURL: URL + ): Promise> { + const entry = await this.resolveConfigForFile(fileURL); + return entry.config.connections; + } + + public async getConfigForFile(fileURL: URL): Promise { + const entry = await this.resolveConfigForFile(fileURL); + return entry.config; + } + + public async getEffectiveConfigSource(fileURL: URL): Promise<{ + source: 'discovered' | 'global' | 'defaults'; + configFileUri?: string; + }> { + const entry = await this.resolveConfigForFile(fileURL); + return { + source: entry.source, + configFileUri: entry.configURL?.toString(), + }; + } + + // --- Private implementation --- + + private invalidateCache(): void { + for (const entry of this.configCache.values()) { + entry.config.releaseConnections().catch(err => { + console.warn('Error releasing config connections:', err); + }); + } + this.configCache.clear(); + this.directoryIndex.clear(); + } + + private workspaceFolderFor(fileURL: URL): URL { + const candidates = this.workspaceRoots + .filter(r => isAncestor(r, fileURL)) + .sort((a, b) => b.toString().length - a.toString().length); + return candidates[0] ?? new URL('.', fileURL); + } + + private async resolveConfigForFile(fileURL: URL): Promise { + const workspaceFolder = this.workspaceFolderFor(fileURL); + const fileDir = new URL('.', fileURL).toString(); + + const indexedKey = this.directoryIndex.get(fileDir); + if (indexedKey !== undefined) { + const cached = this.configCache.get(indexedKey); + if (cached) return cached; + this.directoryIndex.delete(fileDir); + } + + // Level 1 — discovery (project config) + const discovered = await this.tryDiscovery(fileURL, workspaceFolder); + if (discovered) { + const identityKey = `discovered:${workspaceFolder}:${ + discovered.configURL?.toString() ?? '' + }`; + + const existing = this.configCache.get(identityKey); + if (existing) { + this.directoryIndex.set(fileDir, identityKey); + return existing; + } + + this.applyWrappers(discovered.config, {workspaceFolder}); + const entry: CachedConfig = { + config: discovered.config, + source: 'discovered', + configURL: discovered.configURL, + }; + this.configCache.set(identityKey, entry); + this.directoryIndex.set(fileDir, identityKey); + return entry; + } + + const fallbackKey = `fallback:${workspaceFolder}`; + const existingFallback = this.configCache.get(fallbackKey); + if (existingFallback) { + this.directoryIndex.set(fileDir, fallbackKey); + return existingFallback; + } + + // Level 2 — global config + if (this.globalConfigDirectory) { + const global = await this.tryGlobalConfig(workspaceFolder); + if (global) { + this.applyWrappers(global.config, {workspaceFolder}); + const entry: CachedConfig = { + config: global.config, + source: 'global', + configURL: global.configURL, + }; + this.configCache.set(fallbackKey, entry); + this.directoryIndex.set(fileDir, fallbackKey); + return entry; + } + } + + // Level 3 — settings + defaults + const built = this.buildSettingsAndDefaultsConfig(workspaceFolder); + this.applyWrappers(built.config, {workspaceFolder}); + const entry: CachedConfig = { + config: built.config, + source: 'defaults', + }; + this.configCache.set(fallbackKey, entry); + this.directoryIndex.set(fileDir, fallbackKey); + return entry; + } + + private async tryDiscovery( + fileURL: URL, + workspaceFolder: URL + ): Promise<{config: MalloyConfig; configURL?: URL} | undefined> { + if (!this.urlReader) return undefined; + try { + const config = await discoverConfig( + fileURL, + workspaceFolder, + this.urlReader + ); + if (!config) return undefined; + const rawURL = await config.readOverlay('config', 'configURL'); + const configURL = + typeof rawURL === 'string' ? new URL(rawURL) : undefined; + return {config, configURL}; + } catch (err) { + console.warn(`Malloy config discovery failed near ${fileURL}:`, err); + return undefined; + } + } + + private secretOverlay(): Overlay | undefined { + const resolver = this.secretResolver; + if (!resolver) return undefined; + return async (path: string[]) => { + if (path.length !== 1) return undefined; + if (!SECRET_KEY_PATTERN.test(path[0])) return undefined; + return await resolver(path[0]); + }; + } + + private async tryGlobalConfig( + workspaceFolder: URL + ): Promise<{config: MalloyConfig; configURL?: URL} | undefined> { + if (!this.urlReader) return undefined; + if (!this.hostAdapter.expandHome || !this.hostAdapter.pathToFileURL) { + return undefined; + } + + const expandedDir = this.hostAdapter.expandHome(this.globalConfigDirectory); + if (!expandedDir) return undefined; + + let dirUrl = this.hostAdapter.pathToFileURL( + expandedDir.endsWith('/') || expandedDir.endsWith('\\') + ? expandedDir + : expandedDir + '/' + ); + if (!dirUrl.href.endsWith('/')) { + dirUrl = new URL(dirUrl.href + '/'); + } + const globalConfigURL = new URL('malloy-config.json', dirUrl); + + let text: string; + try { + const result = await this.urlReader.readURL(globalConfigURL); + text = typeof result === 'string' ? result : result.contents; + } catch { + return undefined; + } + + let pojo: object; + try { + pojo = JSON.parse(text); + } catch (err) { + console.warn( + `Malloy global config invalid JSON at ${globalConfigURL}:`, + err + ); + return undefined; + } + + const overlays: ConfigOverlays = { + config: contextOverlay({ + rootDirectory: workspaceFolder.toString(), + configURL: globalConfigURL.toString(), + }), + }; + return { + config: new MalloyConfig(pojo, overlays), + configURL: globalConfigURL, + }; + } + + private buildSettingsAndDefaultsConfig(workspaceFolder: URL): { + config: MalloyConfig; + configURL?: undefined; + } { + const overlays: ConfigOverlays = { + config: contextOverlay({ + rootDirectory: workspaceFolder.toString(), + }), + }; + const secret = this.secretOverlay(); + if (secret) overlays['secret'] = secret; + const pojo: { + includeDefaultConnections: boolean; + connections?: Record; + } = {includeDefaultConnections: true}; + if (this.settingsConfig) { + pojo.connections = translateSettingsEntries(this.settingsConfig); + } + const config = new MalloyConfig(pojo, overlays); + return {config}; + } + + private applyWrappers( + config: MalloyConfig, + opts: {workspaceFolder: URL} + ): void { + const postProcess = this.connectionFactory.postProcessConnection?.bind( + this.connectionFactory + ); + if (postProcess) { + const workingDir = opts.workspaceFolder.toString(); + config.wrapConnections(base => ({ + lookupConnection: async (name: string): Promise => { + const conn = await base.lookupConnection(name); + postProcess(conn, workingDir); + return conn; + }, + })); + } + } + + static getDefaultConnectionTypes(): Record { + const registeredTypes = getRegisteredConnectionTypes(); + + if ( + registeredTypes.includes('duckdb_wasm') && + !registeredTypes.includes('duckdb') + ) { + return {duckdb: 'duckdb_wasm'}; + } + + const defaults: Record = {}; + for (const typeName of registeredTypes) { + defaults[typeName] = typeName; + } + defaults['md'] = 'duckdb'; + return defaults; + } +} diff --git a/packages/malloy-language-server/src/common/connections/types.ts b/packages/malloy-language-server/src/common/connections/types.ts new file mode 100644 index 000000000..df03cebc5 --- /dev/null +++ b/packages/malloy-language-server/src/common/connections/types.ts @@ -0,0 +1,10 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type {Connection} from '@malloydata/malloy'; + +export interface ConnectionFactory { + postProcessConnection?(conn: Connection, workingDir: string): void; +} diff --git a/packages/malloy-language-server/src/common/errors.ts b/packages/malloy-language-server/src/common/errors.ts new file mode 100644 index 000000000..b1504dadc --- /dev/null +++ b/packages/malloy-language-server/src/common/errors.ts @@ -0,0 +1,32 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +export const errorMessage = (error: unknown): string => { + let message = ''; + if (error instanceof Error) { + message = error.message; + } else if (typeof error === 'string') { + message = error; + } else if (error && typeof error === 'object') { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const obj = error as any; + if (typeof obj.message === 'string' && obj.message) { + message = obj.message; + } else if (typeof obj.data === 'string' && obj.data) { + message = obj.data; + } else { + try { + message = JSON.stringify(error); + } catch { + // ignore + } + } + } + if (!message) { + console.error('Unknown error object:', error); + message = 'Something went wrong'; + } + return message; +}; diff --git a/packages/malloy-language-server/src/common/log.ts b/packages/malloy-language-server/src/common/log.ts new file mode 100644 index 000000000..fdfc251f2 --- /dev/null +++ b/packages/malloy-language-server/src/common/log.ts @@ -0,0 +1,29 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type {InvalidationKey} from '@malloydata/malloy'; + +export const prettyLogUri = (uri: string): string => { + const [base, hash] = uri.split('#'); + + let pretty = base.split('/').pop() || ''; + if (hash) { + const match = /^W(\d+)s(.*)$/.exec(hash); + if (match) { + pretty += `:${match[1]}`; + } + } + return pretty; +}; + +export const prettyLogInvalidationKey = ( + invalidationKey: InvalidationKey +): string => { + return `v(${`${invalidationKey}`.substring(0, 8)})`; +}; + +export const prettyTime = (ms: number): string => { + return (ms / 1000).toFixed(3) + 's'; +}; diff --git a/packages/malloy-language-server/src/common/malloy_sql.ts b/packages/malloy-language-server/src/common/malloy_sql.ts new file mode 100644 index 000000000..bc3166572 --- /dev/null +++ b/packages/malloy-language-server/src/common/malloy_sql.ts @@ -0,0 +1,37 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type {LogMessage} from '@malloydata/malloy'; +import type {EmbeddedMalloyQuery} from '@malloydata/malloy-sql'; + +export const fixLogRange = ( + uri: string, + malloyQuery: EmbeddedMalloyQuery, + log: LogMessage, + adjustment = 0 +): string => { + if (log.at) { + if (log.at.url === 'internal://internal.malloy') { + log.at.url = uri; + } else if (log.at.url !== uri) { + return ''; + } + + const embeddedStart: number = log.at.range.start.line + adjustment; + if (embeddedStart === 0) { + log.at.range.start.character += malloyQuery.malloyRange.start.character; + if (log.at.range.start.line === log.at.range.end.line) + log.at.range.end.character += malloyQuery.malloyRange.start.character; + } + + const lineDifference = log.at.range.end.line - log.at.range.start.line; + log.at.range.start.line = malloyQuery.range.start.line + embeddedStart; + log.at.range.end.line = + malloyQuery.range.start.line + embeddedStart + lineDifference; + + return `\n${log.message} at line ${log.at?.range.start.line}`; + } + return ''; +}; diff --git a/packages/malloy-language-server/src/common/schema.ts b/packages/malloy-language-server/src/common/schema.ts new file mode 100644 index 000000000..44c264671 --- /dev/null +++ b/packages/malloy-language-server/src/common/schema.ts @@ -0,0 +1,254 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type { + AtomicTypeDef, + Explore, + Field, + ModelDef, + RepeatedRecordTypeDef, + StructDef, +} from '@malloydata/malloy'; +import {JoinRelationship, isSourceDef} from '@malloydata/malloy'; + +export function isFieldAggregate(field: Field) { + return field.isAtomicField() && field.isCalculation(); +} + +export function fieldType(field: Field) { + if (field.isExplore()) { + if (field.isArray) { + return 'array'; + } else { + return exploreSubtype(field); + } + } else { + return field.isAtomicField() ? field.type.toString() : 'query'; + } +} + +export function exploreSubtype(explore: Explore) { + let subtype; + if (explore.hasParentExplore()) { + const relationship = explore.joinRelationship; + subtype = + relationship === JoinRelationship.ManyToOne + ? 'many_to_one' + : relationship === JoinRelationship.OneToMany + ? 'one_to_many' + : JoinRelationship.OneToOne + ? 'one_to_one' + : 'base'; + } else { + subtype = 'base'; + } + return subtype; +} + +/** + * Cache of compiled field hiding patterns so that for a given schema + * view render, the pattern only needs to be compiled once. Uses a WeakMap + * because the Explore object is typically re-created for each render. + */ +const hiddenFields = new WeakMap< + Explore, + {strings: string[]; pattern?: RegExp} +>(); + +/** + * Guard created because TypeScript wasn't simply treating + * `typeof tag === 'string` as a sufficient guard in filter() + * + * @param tag string | undefined + * @returns true if tag is a string + */ +const isStringTag = (tag: string | undefined): tag is string => + typeof tag === 'string'; + +/** + * Determine whether to hide a field in the schema viewer based on tags + * applied to the source. + * + * `hidden = ["field1", "field2"]` will hide individual fields + * `hidden.pattern = "^_"` will hide fields that match the regular expression + * /^_/. They can be combined. + * + * @param field A Field object + * @returns true if field should not be displayed in schema viewer + */ +export function isFieldHidden(field: Field): boolean { + const {name, parentExplore} = field; + let hidden = hiddenFields.get(parentExplore); + if (!hidden) { + const {tag} = parentExplore.tagParse(); + const strings = + tag + .array('hidden') + ?.map(tag => tag.text()) + .filter(isStringTag) || []; + + const patternText = tag.text('hidden', 'pattern'); + const pattern = patternText ? new RegExp(patternText) : undefined; + + hidden = {strings, pattern}; + hiddenFields.set(field.parentExplore, hidden); + } + return !!(hidden.pattern?.test(name) || hidden.strings.includes(name)); +} + +/** + * Add `` around path elements that have special characters or are in + * the list of reserved words + * @param element A field path element + * @returns A potentially quoted field path element + */ +export const quoteIfNecessary = (element: string) => { + // Quote if contains non-word characters + if (/\W/.test(element) || RESERVED.includes(element.toUpperCase())) { + return `\`${element}\``; + } + return element; +}; + +/** + * Retrieve a source from a model safely + * + * @param modelDef Model definition + * @param sourceName Source name + * @returns SourceDef for given name, or throws if not a source + */ + +export const getSourceDef = (modelDef: ModelDef, sourceName: string) => { + const result = modelDef.contents[sourceName]; + if (isSourceDef(result)) { + return result; + } + throw new Error(`Not a source: ${sourceName}`); +}; + +/* + * It would be nice if these types made it out of Malloy, or if this + * functionality moved into core Malloy + */ + +interface NativeUnsupportedTypeDef { + type: 'sql native'; + rawType?: string; +} + +interface RecordElementTypeDef { + type: 'record_element'; +} + +type TypeDef = + | RepeatedRecordTypeDef + | AtomicTypeDef + | NativeUnsupportedTypeDef + | RecordElementTypeDef; + +export const getTypeLabelFromStructDef = (structDef: StructDef): string => { + const getTypeLabelFromTypeDef = (typeDef: TypeDef): string => { + if (typeDef.type === 'array') { + return `${getTypeLabelFromTypeDef(typeDef.elementTypeDef)}[]`; + } + if (typeDef.type === 'sql native' && typeDef.rawType) { + return `${typeDef.type} (${typeDef.rawType})`; + } + return typeDef.type; + }; + + if (structDef.type === 'array') { + return `${getTypeLabelFromTypeDef(structDef.elementTypeDef)}[]`; + } + return structDef.type; +}; + +export const getTypeLabel = (field: Field): string => { + if (field.isExplore()) { + if (field.isArray) { + return getTypeLabelFromStructDef(field.structDef); + } else { + return ''; + } + } + let typeLabel = fieldType(field); + if (field.isAtomicField() && field.isUnsupported()) { + typeLabel = `${typeLabel} (${field.rawType})`; + } + return typeLabel; +}; + +const RESERVED: string[] = [ + 'ALL', + 'AND', + 'AS', + 'ASC', + 'AVG', + 'BOOLEAN', + 'BY', + 'CASE', + 'CAST', + 'CONDITION', + 'COUNT', + 'DATE', + 'DAY', + 'DAYS', + 'DESC', + 'DISTINCT', + 'ELSE', + 'END', + 'EXCLUDE', + 'EXTEND', + 'FALSE', + 'FULL', + 'FOR', + 'FROM', + 'FROM_SQL', + 'HAS', + 'HOUR', + 'HOURS', + 'IMPORT', + 'INNER', + 'IS', + 'JSON', + 'LAST', + 'LEFT', + 'MAX', + 'MIN', + 'MINUTE', + 'MINUTES', + 'MONTH', + 'MONTHS', + 'NOT', + 'NOW', + 'NULL', + 'NUMBER', + 'ON', + 'OR', + 'PICK', + 'QUARTER', + 'QUARTERS', + 'RIGHT', + 'SECOND', + 'SECONDS', + 'STRING', + 'SOURCE_KW', + 'SUM', + 'SQL', + 'TABLE', + 'THEN', + 'THIS', + 'TIMESTAMP', + 'TO', + 'TRUE', + 'TURTLE', + 'WEEK', + 'WEEKS', + 'WHEN', + 'WITH', + 'YEAR', + 'YEARS', + 'UNGROUPED', +] as const; diff --git a/packages/malloy-language-server/src/common/types/connection_manager_types.ts b/packages/malloy-language-server/src/common/types/connection_manager_types.ts new file mode 100644 index 000000000..68e2d1dd9 --- /dev/null +++ b/packages/malloy-language-server/src/common/types/connection_manager_types.ts @@ -0,0 +1,38 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type { + Connection, + JsonConfigValue, + LookupConnection, + MalloyConfig, +} from '@malloydata/malloy'; + +export type UnresolvedConnectionConfigEntry = { + is: string; + [key: string]: + | string + | number + | boolean + | JsonConfigValue + | {env: string} + | {secretKey: string} + | undefined; +}; + +export interface ConnectionConfigManager { + getConnectionsConfig(): + | Record + | undefined; + onConfigurationUpdated(): Promise; +} + +export interface ConnectionManager { + getConnectionLookup(fileURL: URL): Promise>; + setConnectionsConfig( + connectionsConfig: Record + ): void; + getConfigForFile(fileURL: URL): Promise; +} diff --git a/packages/malloy-language-server/src/common/types/file_handler.ts b/packages/malloy-language-server/src/common/types/file_handler.ts new file mode 100644 index 000000000..fb15c7785 --- /dev/null +++ b/packages/malloy-language-server/src/common/types/file_handler.ts @@ -0,0 +1,33 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +export interface CellMetadataConfig { + [key: string]: unknown; + connection?: string; +} + +export interface CellMetadata { + [key: string]: unknown; + config?: CellMetadataConfig; +} + +export interface Cell { + uri: string; + text: string; + languageId: string; + metadata?: CellMetadata; + lineOffset: number; +} + +export interface CellData { + baseUri: string; + cells: Cell[]; +} + +export interface BuildModelRequest { + uri: string; + languageId: string; + refreshSchemaCache?: boolean; +} diff --git a/packages/malloy-language-server/src/common/types/query_spec.ts b/packages/malloy-language-server/src/common/types/query_spec.ts new file mode 100644 index 000000000..023ed5f5f --- /dev/null +++ b/packages/malloy-language-server/src/common/types/query_spec.ts @@ -0,0 +1,11 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +export interface DocumentMetadata { + fileName: string; + uri: string; + languageId: string; + version: number; +} diff --git a/packages/malloy-language-server/src/completions/completions.ts b/packages/malloy-language-server/src/completions/completions.ts new file mode 100644 index 000000000..4dde2a868 --- /dev/null +++ b/packages/malloy-language-server/src/completions/completions.ts @@ -0,0 +1,59 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type {CompletionParams, CompletionItem} from 'vscode-languageserver'; +import {CompletionItemKind, MarkupKind} from 'vscode-languageserver'; +import type {TextDocument} from 'vscode-languageserver-textdocument'; +import {COMPLETION_DOCS} from '../common/completion_docs'; +import {parseWithCache} from '../parse_cache'; +import type {TranslateCache} from '../translate_cache'; +import {getSchemaCompletions} from './schema_completions'; + +export async function getCompletionItems( + document: TextDocument, + context: CompletionParams, + translateCache: TranslateCache +): Promise { + const schemaCompletions = await getSchemaCompletions( + document, + context, + translateCache + ); + if (schemaCompletions) { + return schemaCompletions.map(completion => { + return { + kind: CompletionItemKind.Field, + label: completion, + }; + }); + } + const completions = parseWithCache(document).completions(context.position); + const cleanedCompletions: CompletionItem[] = completions.map(completion => { + return { + kind: CompletionItemKind.Property, + label: completion.text, + data: { + type: completion.type, + property: completion.text.substring(0, completion.text.length - 2), + }, + }; + }); + return cleanedCompletions; +} + +export function resolveCompletionItem(item: CompletionItem): CompletionItem { + if (item.data) { + const data = item.data; + item.detail = data.property; + const docs = (COMPLETION_DOCS[data.type] || {})[data.property]; + if (docs) { + item.documentation = { + kind: MarkupKind.Markdown, + value: docs, + }; + } + } + return item; +} diff --git a/packages/malloy-language-server/src/completions/schema_completions.ts b/packages/malloy-language-server/src/completions/schema_completions.ts new file mode 100644 index 000000000..c40d77e72 --- /dev/null +++ b/packages/malloy-language-server/src/completions/schema_completions.ts @@ -0,0 +1,301 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type {CompletionParams} from 'vscode-languageserver'; +import type {Position, TextDocument} from 'vscode-languageserver-textdocument'; +import type {TranslateCache} from '../translate_cache'; +import type {Explore, Field, Model} from '@malloydata/malloy'; +import {fieldType, isFieldAggregate} from '../common/schema'; + +const QUERY_KEYWORDS = [ + 'aggregate', + 'calculate', + 'group_by', + 'having', + 'index', + 'limit', + 'order_by', + 'select', + 'sample', + 'top', + 'where', + 'timezone', +]; + +const DIMENSION_KEYWORDS = [ + 'select', + 'group_by', + 'order_by', + 'aggregate', + 'having', + 'order_by', +]; + +const MEASURE_KEYWORDS = ['aggregate', 'having']; + +const EXPLORE_KEYWORDS = [ + 'select', + 'group_by', + 'order_by', + 'aggregate', + 'having', + 'order_by', +]; + +const UNNAMED_QUERY = + /\b(?:run|query)\s*:\s*([A-Za-z_][A-Za-z_0-9]*)\s*->\s*{/i; + +const NAMED_QUERY = + /\bquery\s*:\s*(?:[A-Za-z_][A-Za-z_0-9]*)\s+is\s+([A-Za-z_][A-Za-z_0-9]*)\s+->\s*{/i; + +const FIELD_CHAIN = /\s(?:[a-zA-Z_][.A-Za-z_0-9]*)?$/; + +const TOP_LEVEL_SYMBOLS = /\s*((?:source|query|run|sql)\s*:|import)\s*/i; + +const LINE_COMMENTS = /\s*(--|\/\/)/g; + +export async function getSchemaCompletions( + document: TextDocument, + context: CompletionParams, + translateCache: TranslateCache +): Promise { + const parse = parseDocumentText(document.getText(), context.position); + if (parse.nearestDefinitionText && parse.adjustedCursor) { + const exploreName = getQueriedExploreName(parse.nearestDefinitionText); + const {keyword, fieldChain} = getQueryContext( + parse.nearestDefinitionText, + parse.adjustedCursor + ); + if (exploreName !== null && keyword !== null && fieldChain !== null) { + let model: Model | undefined = undefined; + try { + model = await translateCache.translateWithTruncatedCache( + document, + parse.truncatedText, + parse.exploreCount + ); + } catch (error: unknown) { + console.error( + `Error fetching model for document sources and imports '${document.uri}': ${error}` + ); + } + if (model) { + const exploreMap: Record = {}; + model.explores.forEach(explore => { + exploreMap[explore.name] = explore; + }); + const explore = exploreMap[exploreName]; + if (explore) { + const fields = getEligibleFields(fieldChain, explore); + return filterCompletions(fields, keyword, fieldChain); + } + } + } + } + return []; +} + +function getQueriedExploreName(text: string[]): string | null { + const unnamedQueryMatch = UNNAMED_QUERY.exec(text[0]); + if (unnamedQueryMatch) { + return unnamedQueryMatch[1]; + } + const namedQueryMatch = NAMED_QUERY.exec(text[0]); + return namedQueryMatch ? namedQueryMatch[1] : null; +} + +function getQueryContext(lines: string[], cursor: Position) { + lines[cursor.line] = lines[cursor.line].slice(0, cursor.character); + const fieldChainMatch = FIELD_CHAIN.exec(lines[cursor.line]); + const fieldChain: string | null = fieldChainMatch + ? fieldChainMatch[0].trim() + : null; + let i = cursor.line; + let keyword: string | null = null; + + if (fieldChain === null) { + return {keyword, fieldChain}; + } + + while (i >= 0) { + const currentLine = lines[i]; + const queryKeywords = new RegExp( + `\\b(?:${QUERY_KEYWORDS.join('|')}):`, + 'gi' + ); + const matches = currentLine.match(queryKeywords); + if (matches) { + keyword = matches[matches.length - 1] + .slice(0, matches[matches.length - 1].length - 1) + .toLowerCase(); + break; + } + i--; + } + return {keyword, fieldChain}; +} + +function getEligibleFields(fieldChain: string, explore: Explore): Field[] { + const fieldTree = fieldChain.split('.'); + fieldTree.pop(); + let currentExplore: Explore | undefined = explore; + for (const fieldName of fieldTree) { + if (fieldName.length === 0) { + return []; + } + let validField = false; + if (currentExplore) { + for (const field of currentExplore.allFields) { + if (fieldName === field.name && field.isExploreField()) { + validField = true; + currentExplore = field; + break; + } + } + } + if (!validField) { + return []; + } + } + return currentExplore?.allFields || []; +} + +function filterCompletions( + fields: Field[], + keyword: string, + fieldChain: string +) { + let completions: string[] = []; + const fieldTree = fieldChain.split('.'); + const eligibleFieldsPrefix = fieldTree[fieldTree.length - 1]; + const {dimensions, measures, explores} = bucketMatchingFieldNames( + fields, + eligibleFieldsPrefix + ); + if (DIMENSION_KEYWORDS.includes(keyword)) { + completions = [...completions, ...dimensions]; + } + if (MEASURE_KEYWORDS.includes(keyword)) { + completions = [...completions, ...measures]; + } + if (EXPLORE_KEYWORDS.includes(keyword)) { + completions = [...completions, ...explores]; + } + return completions; +} + +function bucketMatchingFieldNames(fields: Field[], fieldToMatch: string) { + const queries: string[] = []; + const dimensions: string[] = []; + const measures: string[] = []; + const explores: string[] = []; + for (const field of fields) { + if (fieldToMatch.length === 0 || field.name.startsWith(fieldToMatch)) { + const type = fieldType(field); + if (isFieldAggregate(field)) { + measures.push(field.name); + } else if (field.isExploreField()) { + explores.push(field.name); + } else if (type === 'query') { + queries.push(field.name); + } else { + dimensions.push(field.name); + } + } + } + return {queries, dimensions, measures, explores}; +} + +export interface DocumentTextParse { + truncatedText: string; + exploreCount: number; + nearestDefinitionText?: string[]; + adjustedCursor?: Position; +} + +export function parseDocumentText( + text: string, + cursor: Position +): DocumentTextParse { + const parse: DocumentTextParse = { + truncatedText: '', + exploreCount: 0, + }; + const truncatedLines: string[] = []; + + let lastSymbolStart: Position | undefined = undefined; + let lastNonemptyLine = -1; + let includeLines = false; + + const lines = text.split('\n'); + for (const [i, line] of lines.entries()) { + const symbolMatches = TOP_LEVEL_SYMBOLS.exec(line); + if (symbolMatches !== null && symbolMatches.index === 0) { + if ( + symbolMatches[1].toLowerCase().startsWith('source') || + symbolMatches[1].toLowerCase().startsWith('import') + ) { + includeLines = true; + parse.exploreCount++; + } else { + includeLines = false; + } + parseCurrentDefinition( + lastSymbolStart, + cursor, + lastNonemptyLine, + parse, + lines + ); + lastSymbolStart = { + line: i, + character: symbolMatches.index, + }; + } + if (includeLines) { + truncatedLines.push(line); + } + if (!isEmptyLine(line)) { + lastNonemptyLine = i; + } + } + parseCurrentDefinition( + lastSymbolStart, + cursor, + lastNonemptyLine, + parse, + lines + ); + parse.truncatedText = truncatedLines.join('\n'); + return parse; +} + +function parseCurrentDefinition( + lastSymbolStart: Position | undefined, + cursor: Position, + lastNonemptyLine: number, + parse: DocumentTextParse, + lines: string[] +) { + if ( + lastSymbolStart && + cursor.line >= lastSymbolStart.line && + cursor.line <= lastNonemptyLine + ) { + parse.nearestDefinitionText = lines.slice( + lastSymbolStart.line, + lastNonemptyLine + 1 + ); + parse.adjustedCursor = { + line: cursor.line - lastSymbolStart.line, + character: cursor.character, + }; + } +} + +function isEmptyLine(line: string) { + const commentMatches = LINE_COMMENTS.exec(line); + return (commentMatches && commentMatches.index === 0) || line.length === 0; +} diff --git a/packages/malloy-language-server/src/create_server.ts b/packages/malloy-language-server/src/create_server.ts new file mode 100644 index 000000000..987f952b1 --- /dev/null +++ b/packages/malloy-language-server/src/create_server.ts @@ -0,0 +1,345 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type { + InitializeParams, + InitializeResult, + CompletionItem, + HoverParams, + Hover, + Connection, + Position, + DidChangeConfigurationParams, +} from 'vscode-languageserver'; +import {TextDocuments, TextDocumentSyncKind} from 'vscode-languageserver'; +import debounce from 'lodash/debounce'; + +import {TextDocument} from 'vscode-languageserver-textdocument'; +import type {URLReader} from '@malloydata/malloy'; +import { + getMalloyDiagnostics, + aggregateNotebookDiagnostics, +} from './diagnostics'; +import {getMalloySymbols} from './symbols'; +import {getMalloyLenses} from './lenses'; +import { + getCompletionItems, + resolveCompletionItem, +} from './completions/completions'; +import {getHover} from './hover/hover'; +import {getMalloyDefinitionReference} from './definitions/definitions'; +import type { + CellDataProvider, + WorkspaceFolderProvider, +} from './translate_cache'; +import {TranslateCache} from './translate_cache'; +import type {CommonConnectionManager} from './common/connection_manager'; +import {findMalloyLensesAt} from './lenses/lenses'; +import {prettyLogUri} from './common/log'; +import {getMalloyCodeAction} from './code_actions/code_actions'; + +export interface CreateServerOptions { + onDidChangeConfiguration?: (params: DidChangeConfigurationParams) => void; + urlReader?: URLReader; + cellDataProvider?: CellDataProvider; + workspaceFolderProvider?: WorkspaceFolderProvider; +} + +export const createServer = ( + connection: Connection, + connectionManager: CommonConnectionManager, + options?: CreateServerOptions +) => { + const urlReader = options?.urlReader ?? connectionManager.getURLReader(); + if (!urlReader) { + throw new Error( + 'A URLReader must be provided via options.urlReader or connectionManager.setURLReader()' + ); + } + + const documents = new TextDocuments(TextDocument); + connection.onInitialize((params: InitializeParams) => { + connection.console.info('onInitialize'); + + const result: InitializeResult = { + capabilities: { + textDocumentSync: TextDocumentSyncKind.Incremental, + documentSymbolProvider: true, + codeLensProvider: { + resolveProvider: false, + }, + completionProvider: { + resolveProvider: true, + }, + codeActionProvider: { + resolveProvider: false, + }, + definitionProvider: true, + hoverProvider: true, + }, + }; + + if (params.capabilities.workspace?.workspaceFolders) { + result.capabilities.workspace = { + workspaceFolders: { + supported: true, + }, + }; + } + + if (params.workspaceFolders) { + connectionManager.setWorkspaceRoots( + params.workspaceFolders.map(f => f.uri) + ); + } + + return result; + }); + + const logger = { + info: (msg: string) => connection.console.info(msg), + debug: (msg: string) => connection.console.log(msg), + error: (msg: string) => connection.console.error(msg), + }; + + const translateCache = new TranslateCache( + documents, + logger, + connectionManager, + urlReader, + options?.cellDataProvider, + options?.workspaceFolderProvider, + connection + ); + + async function diagnoseDocument(document: TextDocument) { + const prettyUri = prettyLogUri(document.uri); + + connection.console.info(`diagnoseDocument ${prettyUri} start`); + const versionsAtRequestTime = new Map( + documents.all().map(document => [document.uri, document.version]) + ); + const diagnostics = await getMalloyDiagnostics(translateCache, document); + + for (const uri in diagnostics) { + const versionAtRequest = versionsAtRequestTime.get(uri); + const currentVersion = documents.get(uri)?.version; + if ( + versionAtRequest === undefined || + currentVersion === versionAtRequest + ) { + await connection.sendDiagnostics({ + uri, + diagnostics: diagnostics[uri], + version: currentVersion, + }); + } + } + + try { + const notebookDiagnostics = await aggregateNotebookDiagnostics( + diagnostics, + translateCache + ); + for (const notebookUri in notebookDiagnostics) { + await connection.sendDiagnostics({ + uri: notebookUri, + diagnostics: notebookDiagnostics[notebookUri], + }); + } + } catch (error) { + connection.console.error( + `Failed to aggregate notebook diagnostics: ${error}` + ); + } + + for (const dependency of translateCache.dependentsOf(document.uri) ?? []) { + const document = documents.get(dependency); + if (document) { + connection.console.info( + `diagnoseDocument recompiling ${prettyLogUri(document.uri)}` + ); + debouncedDiagnoseDocument(document); + } + } + connection.console.info(`diagnoseDocument ${prettyUri} end`); + } + + const debouncedDiagnoseDocuments: Record< + string, + (document: TextDocument) => Promise | undefined + > = {}; + + const debouncedDiagnoseDocument = (document: TextDocument) => { + const {uri} = document; + if (!debouncedDiagnoseDocuments[uri]) { + debouncedDiagnoseDocuments[uri] = debounce(diagnoseDocument, 300); + } + debouncedDiagnoseDocuments[uri](document)?.catch(console.error); + }; + + documents.onDidChangeContent(change => { + debouncedDiagnoseDocument(change.document); + }); + + function ejectIfUnused(uri: string) { + const dependents = translateCache.dependentsOf(uri); + if (dependents && dependents.length === 0) { + connection.console.info(`ejectIfUnused ejecting ${prettyLogUri(uri)}`); + const dependencies = translateCache.dependenciesFor(uri) ?? []; + translateCache.deleteModel(uri); + for (const other of dependencies) { + const document = documents.get(other); + if (document === undefined) { + connection.console.info( + `ejectIfUnused no document for ${prettyLogUri( + other + )}, considering deletion` + ); + ejectIfUnused(other); + } + } + } + } + + documents.onDidClose(event => { + const {uri} = event.document; + ejectIfUnused(uri); + delete debouncedDiagnoseDocuments[uri]; + }); + + connection.onDocumentSymbol(handler => { + const document = documents.get(handler.textDocument.uri); + if (document && document.languageId === 'malloy') { + try { + return getMalloySymbols(document); + } catch (error) { + console.error('getMalloySymbols', error); + } + } + return []; + }); + + connection.onCodeLens(async handler => { + const document = documents.get(handler.textDocument.uri); + if (document && document.languageId === 'malloy') { + try { + return await getMalloyLenses(connection, document, connectionManager); + } catch (error) { + console.error('getMalloyLenses', error); + } + } + return []; + }); + + connection.onCodeAction(async handler => { + const document = documents.get(handler.textDocument.uri); + if (document) { + return getMalloyCodeAction(translateCache, document, handler.range); + } + return null; + }); + + connection.onRequest( + 'malloy/findLensesAt', + async ({uri, position}: {uri: string; position: Position}) => { + const document = documents.get(uri); + if (document && position) { + try { + return await findMalloyLensesAt( + connection, + document, + position, + connectionManager + ); + } catch (error) { + console.error('findMalloyLensesAt', error); + } + } + return []; + } + ); + + connection.onDefinition(handler => { + const document = documents.get(handler.textDocument.uri); + if (document && document.languageId === 'malloy') { + try { + return getMalloyDefinitionReference( + translateCache, + document, + handler.position + ); + } catch (error) { + console.error('getMalloyDefinitionReference', error); + } + } + return []; + }); + + connection.onDidChangeConfiguration(change => { + options?.onDidChangeConfiguration?.(change); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const settings = (change?.settings as any)?.malloy ?? {}; + connectionManager.setConnectionsConfig(settings.connectionMap ?? {}); + connectionManager.setGlobalConfigDirectory( + settings.globalConfigDirectory ?? '' + ); + translateCache.deleteAllModels(); + documents.all().forEach(debouncedDiagnoseDocument); + }); + + connection.onRequest( + 'malloy/getEffectiveConfigSource', + async ({fileUri}: {fileUri: string}) => { + return connectionManager.getEffectiveConfigSource(new URL(fileUri)); + } + ); + + connection.onRequest('malloy/invalidateConnectionCache', () => { + connectionManager.notifyConfigFileChanged(); + translateCache.deleteAllModels(); + documents.all().forEach(debouncedDiagnoseDocument); + }); + + connection.onCompletion(async (params): Promise => { + const document = documents.get(params.textDocument.uri); + if (document && document.languageId === 'malloy') { + try { + const completionItems = await getCompletionItems( + document, + params, + translateCache + ); + return completionItems; + } catch (error) { + console.error('getCompletionItems', error); + } + } + return []; + }); + + connection.onCompletionResolve((item: CompletionItem): CompletionItem => { + return resolveCompletionItem(item); + }); + + connection.onHover(async (params: HoverParams): Promise => { + const document = documents.get(params.textDocument.uri); + + if (document && document.languageId === 'malloy') { + try { + return getHover(document, documents, translateCache, params); + } catch (error) { + console.error('getHover', error); + } + } + return null; + }); + + documents.listen(connection); + + connection.listen(); + + connection.console.info('Server loaded'); +}; diff --git a/packages/malloy-language-server/src/definitions/definitions.ts b/packages/malloy-language-server/src/definitions/definitions.ts new file mode 100644 index 000000000..73231cde2 --- /dev/null +++ b/packages/malloy-language-server/src/definitions/definitions.ts @@ -0,0 +1,53 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type {Location, Position, DefinitionLink} from 'vscode-languageserver'; +import type {TextDocument} from 'vscode-languageserver-textdocument'; +import type {TranslateCache} from '../translate_cache'; + +export async function getMalloyDefinitionReference( + translateCache: TranslateCache, + document: TextDocument, + position: Position +): Promise { + try { + const model = await translateCache.translateWithCache( + document.uri, + document.languageId + ); + if (!model) { + return []; + } + const reference = model.getReference(position); + const location = reference?.definition.location; + if (location) { + return [ + { + uri: location.url, + range: location.range, + }, + ]; + } else { + const importLocation = model.getImport(position); + if (importLocation) { + const documentStart = { + start: {line: 0, character: 0}, + end: {line: 0, character: 0}, + }; + return [ + { + originSelectionRange: importLocation.location.range, + targetUri: importLocation.importURL, + targetRange: documentStart, + targetSelectionRange: documentStart, + }, + ]; + } + } + return []; + } catch (error) { + return []; + } +} diff --git a/packages/malloy-language-server/src/diagnostics/diagnostics.ts b/packages/malloy-language-server/src/diagnostics/diagnostics.ts new file mode 100644 index 000000000..a73dff966 --- /dev/null +++ b/packages/malloy-language-server/src/diagnostics/diagnostics.ts @@ -0,0 +1,155 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type {Diagnostic} from 'vscode-languageserver'; +import {DiagnosticSeverity} from 'vscode-languageserver'; +import type {LogMessage} from '@malloydata/malloy'; +import {MalloyError} from '@malloydata/malloy'; +import type {TextDocument} from 'vscode-languageserver-textdocument'; +import type {TranslateCache} from '../translate_cache'; +import {parseMalloySQLSQLWithCache} from '../parse_cache'; +import {errorMessage} from '../common/errors'; +import {prettyLogUri} from '../common/log'; +import {getRenderTagDiagnostics} from './render-tag-validation'; +const errorDictURI = + 'https://docs.malloydata.dev/documentation/error_dictionary'; + +const DEFAULT_RANGE = { + start: {line: 0, character: 0}, + end: {line: 0, character: Number.MAX_VALUE}, +}; + +export async function getMalloyDiagnostics( + translateCache: TranslateCache, + document: TextDocument +): Promise<{[uri: string]: Diagnostic[]}> { + const byURI: {[uri: string]: Diagnostic[]} = { + [document.uri]: [], + }; + const problems: LogMessage[] = []; + + if (document.languageId === 'malloy-sql') { + const {errors} = parseMalloySQLSQLWithCache(document); + if (errors) errors.forEach(e => problems.push(...e.problems)); + } + + try { + const model = await translateCache.translateWithCache( + document.uri, + document.languageId + ); + if (model?.problems) { + problems.push(...model.problems); + } + if (model) { + problems.push(...getRenderTagDiagnostics(document, model)); + } + } catch (error) { + if (error instanceof MalloyError) { + problems.push(...error.problems); + } else { + byURI[document.uri].push({ + severity: DiagnosticSeverity.Error, + range: DEFAULT_RANGE, + message: errorMessage(error), + source: 'malloy', + }); + } + } + + for (const problem of problems) { + const sev = + problem.severity === 'warn' + ? DiagnosticSeverity.Warning + : problem.severity === 'debug' + ? DiagnosticSeverity.Information + : DiagnosticSeverity.Error; + + const uri = problem.at ? problem.at.url : document.uri; + + if (byURI[uri] === undefined) { + byURI[uri] = []; + } + + const range = problem.at?.range || DEFAULT_RANGE; + + if (range.start.line >= 0) { + const theDiag: Diagnostic = { + severity: sev, + range, + message: problem.message, + source: 'malloy', + }; + if (problem.errorTag) { + theDiag.code = problem.errorTag; + theDiag.codeDescription = {href: `${errorDictURI}#${theDiag.code}`}; + } + byURI[uri].push(theDiag); + } + } + + console.info( + `getMalloyDiagnostics: ${prettyLogUri(document.uri)} found ${ + problems.length + } problems` + ); + + return byURI; +} + +export async function aggregateNotebookDiagnostics( + diagnostics: {[uri: string]: Diagnostic[]}, + translateCache: TranslateCache +): Promise<{[notebookUri: string]: Diagnostic[]}> { + const result: {[uri: string]: Diagnostic[]} = {}; + + for (const [uri, diags] of Object.entries(diagnostics)) { + let url: URL; + try { + url = new URL(uri); + } catch { + continue; + } + + if (url.protocol !== 'vscode-notebook-cell:') continue; + + const notebookUri = `file://${url.pathname}`; + + if (!result[notebookUri]) { + result[notebookUri] = []; + } + + if (diags.length === 0) continue; + + try { + const cellData = await translateCache.getCellData(url); + const cell = cellData.cells.find(c => c.uri === uri); + const lineOffset = cell?.lineOffset ?? 0; + + const mappedDiags = diags.map(d => ({ + ...d, + range: { + start: { + line: d.range.start.line + lineOffset, + character: d.range.start.character, + }, + end: { + line: d.range.end.line + lineOffset, + character: d.range.end.character, + }, + }, + })); + + result[notebookUri].push(...mappedDiags); + } catch (error) { + console.error( + `Failed to aggregate diagnostics for notebook cell ${uri}:`, + error + ); + } + } + + return result; +} diff --git a/packages/malloy-language-server/src/diagnostics/index.ts b/packages/malloy-language-server/src/diagnostics/index.ts new file mode 100644 index 000000000..a667c7edb --- /dev/null +++ b/packages/malloy-language-server/src/diagnostics/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +export { + getMalloyDiagnostics, + aggregateNotebookDiagnostics, +} from './diagnostics'; diff --git a/packages/malloy-language-server/src/diagnostics/render-tag-validation.ts b/packages/malloy-language-server/src/diagnostics/render-tag-validation.ts new file mode 100644 index 000000000..e39f8a864 --- /dev/null +++ b/packages/malloy-language-server/src/diagnostics/render-tag-validation.ts @@ -0,0 +1,59 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type {LogMessage, Model} from '@malloydata/malloy'; +import {validateRenderTags} from '@malloydata/render-validator'; +import type {TextDocument} from 'vscode-languageserver-textdocument'; +import {parseWithCache} from '../parse_cache'; + +/** + * Validate renderer tag usage for every named and unnamed query in a model. + * + * The renderer's own validator inspects compiled query results — column + * types, tag values, and tag/field-type compatibility — without executing + * the query or needing a DOM. Errors here would otherwise only appear at + * render time in the IDE preview; lifting them into LSP diagnostics + * surfaces them in the editor as the user types. + */ +export function getRenderTagDiagnostics( + document: TextDocument, + model: Model +): LogMessage[] { + if (document.languageId !== 'malloy') return []; + + const out: LogMessage[] = []; + const parse = parseWithCache(document); + let unnamedIndex = 0; + + for (const symbol of parse.symbols) { + if (symbol.type !== 'query' && symbol.type !== 'unnamed_query') continue; + + try { + const preparedQuery = + symbol.type === 'query' + ? model.getPreparedQueryByName(symbol.name) + : model.getPreparedQueryByIndex(unnamedIndex++); + const stableResult = preparedQuery.getPreparedResult().toStableResult(); + const renderLogs = validateRenderTags(stableResult); + + for (const log of renderLogs) { + out.push({ + message: log.message, + severity: log.severity === 'info' ? 'debug' : log.severity, + code: 'render-tag', + errorTag: 'render-tag', + at: log.url + ? {url: log.url, range: log.range} + : {url: document.uri, range: symbol.range.toJSON()}, + }); + } + } catch { + // Query failed to compile; the underlying error is already in + // model.problems, so we skip render validation for this query. + } + } + + return out; +} diff --git a/packages/malloy-language-server/src/hover/hover.ts b/packages/malloy-language-server/src/hover/hover.ts new file mode 100644 index 000000000..cd18102be --- /dev/null +++ b/packages/malloy-language-server/src/hover/hover.ts @@ -0,0 +1,158 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type { + Hover, + HoverParams, + MarkupContent, + TextDocuments, +} from 'vscode-languageserver'; +import {MarkupKind} from 'vscode-languageserver'; +import type {TextDocument} from 'vscode-languageserver-textdocument'; +import type {Annotation, DocumentLocation, Model} from '@malloydata/malloy'; + +import {COMPLETION_DOCS} from '../common/completion_docs'; +import {parseWithCache} from '../parse_cache'; +import type {TranslateCache} from '../translate_cache'; + +type ImportLocation = Exclude, undefined>; +type DocumentReference = Exclude, undefined>; + +export const getHover = async ( + document: TextDocument, + documents: TextDocuments, + translateCache: TranslateCache, + {position}: HoverParams +): Promise => { + const context = parseWithCache(document).helpContext(position); + + if (context?.token) { + const name = context.token.replace(/:$/, ''); + + if (name) { + const value = COMPLETION_DOCS[context.type][name]; + if (value) { + return { + contents: { + kind: MarkupKind.Markdown, + value, + }, + }; + } else if (context.type === 'model_property') { + const model = await translateCache.translateWithCache( + document.uri, + document.languageId + ); + const importLocation = model?.getImport(position); + if (importLocation) { + return getImportHover(translateCache, documents, importLocation); + } + + return null; + } else { + const model = await translateCache.translateWithCache( + document.uri, + document.languageId + ); + const reference = model?.getReference(position); + if (reference) { + return getReferenceHover(translateCache, documents, reference); + } + } + } + } + + return null; +}; + +async function getImportHover( + translateCache: TranslateCache, + documents: TextDocuments, + importLocation: ImportLocation +): Promise { + const importedDocument = documents.get(importLocation.importURL) ?? { + uri: importLocation.importURL, + version: 0, + languageId: 'malloy', + }; + const importedModel = await translateCache.translateWithCache( + importedDocument.uri, + importedDocument.languageId + ); + if (importedModel) { + const sources = bulletedList('Sources', importedModel.exportedExplores); + const queries = bulletedList('Queries', importedModel.namedQueries); + const markdown = `${sources}\n${queries}`; + const contents: MarkupContent = { + kind: MarkupKind.Markdown, + value: markdown, + }; + return { + contents, + }; + } + return null; +} + +function bulletedList( + heading: string, + elements: {name: string; location?: DocumentLocation}[] +): string { + if (elements.length) { + return ( + `* ${heading}\n` + + elements + .map(query => { + const name = query.name; + const uri = query.location + ? `${query.location.url}#${query.location.range.start.line + 1}` + : ''; + return ` * [${name}](${uri})`; + }) + .join('\n') + ); + } + return ''; +} + +function getReferenceHover( + _translateCache: TranslateCache, + _documents: TextDocuments, + {text, definition}: DocumentReference +) { + const tags = annotationToTaglines(definition.annotation).join(''); + const markdown = `\`\`\` +${tags}${text}: ${definition.type} +\`\`\``; + const contents: MarkupContent = { + kind: MarkupKind.Markdown, + value: markdown, + }; + return { + contents, + }; +} + +type NoteArray = Annotation['notes']; + +function annotationToTaglines( + annote: Annotation | undefined, + prefix?: RegExp +): string[] { + annote ||= {}; + const tagLines = annote.inherits + ? annotationToTaglines(annote.inherits, prefix) + : []; + function prefixed(na: NoteArray | undefined): string[] { + const ret: string[] = []; + for (const n of na || []) { + if (prefix === undefined || n.text.match(prefix)) { + ret.push(n.text); + } + } + return ret; + } + return tagLines.concat(prefixed(annote.blockNotes), prefixed(annote.notes)); +} diff --git a/packages/malloy-language-server/src/index.ts b/packages/malloy-language-server/src/index.ts new file mode 100644 index 000000000..a655dc9c9 --- /dev/null +++ b/packages/malloy-language-server/src/index.ts @@ -0,0 +1,40 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +// Main public API +export {createServer} from './create_server'; +export type {CreateServerOptions} from './create_server'; +export {TranslateCache} from './translate_cache'; +export type { + TranslateCacheLogger, + CellDataProvider, + WorkspaceFolderProvider, +} from './translate_cache'; +export {CommonConnectionManager} from './common/connection_manager'; +export type {HostAdapter, SecretResolver} from './common/connection_manager'; +export type {ConnectionFactory} from './common/connections/types'; +export {NodeURLReader} from './node_url_reader'; +export {check} from './check'; + +// Re-export handler functions for direct use +export { + getMalloyDiagnostics, + aggregateNotebookDiagnostics, +} from './diagnostics'; +export {getMalloySymbols} from './symbols'; +export {getMalloyLenses} from './lenses'; +export { + getCompletionItems, + resolveCompletionItem, +} from './completions/completions'; +export {getHover} from './hover/hover'; +export {getMalloyDefinitionReference} from './definitions/definitions'; +export {getMalloyCodeAction} from './code_actions/code_actions'; + +// Re-export types +export type { + ConnectionManager, + UnresolvedConnectionConfigEntry, +} from './common/types/connection_manager_types'; diff --git a/packages/malloy-language-server/src/lenses/index.ts b/packages/malloy-language-server/src/lenses/index.ts new file mode 100644 index 000000000..b9cb5ad6c --- /dev/null +++ b/packages/malloy-language-server/src/lenses/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +export {getMalloyLenses} from './lenses'; diff --git a/packages/malloy-language-server/src/lenses/lenses.ts b/packages/malloy-language-server/src/lenses/lenses.ts new file mode 100644 index 000000000..60a9271b3 --- /dev/null +++ b/packages/malloy-language-server/src/lenses/lenses.ts @@ -0,0 +1,257 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type { + CodeLens, + Connection, + Position, + Range, +} from 'vscode-languageserver'; +import type {TextDocument} from 'vscode-languageserver-textdocument'; +import {parseWithCache} from '../parse_cache'; +import type {ConnectionManager} from '../common/types/connection_manager_types'; +import {getSourceUrl, unquoteIdentifier} from './utils'; +import type {DocumentMetadata} from '../common/types/query_spec'; + +const fixNotebookUrl = async (connection: Connection, url: URL) => { + if (url.protocol === 'vscode-notebook-cell:') { + let protocol = 'file:'; + const workspaceFolders = await connection.workspace.getWorkspaceFolders(); + if (workspaceFolders && workspaceFolders[0]) { + protocol = new URL(workspaceFolders[0].uri).protocol; + } + const {pathname, search} = url; + const host = protocol === 'file:' ? '' : url.host; + const urlString = `${protocol}//${host}${pathname}${search}`; + url = new URL(urlString); + } + + return url; +}; + +export async function getMalloyLenses( + connection: Connection, + document: TextDocument, + connectionManager: ConnectionManager +): Promise { + const lenses: CodeLens[] = []; + const parse = parseWithCache(document); + const symbols = parse.symbols; + const connectionLookup = await connectionManager.getConnectionLookup( + new URL(document.uri) + ); + + const tablepaths = parse.tablePathInfo; + let externalPreview = tablepaths.length ? false : true; + for (const table of tablepaths) { + const conn = await connectionLookup.lookupConnection(table.connectionId); + const tableUrl = await getSourceUrl(table.tablePath, conn); + if (tableUrl) { + lenses.push({ + range: table.range, + command: { + title: 'Table', + command: 'malloy.openUrlInBrowser', + arguments: [tableUrl], + }, + }); + externalPreview = true; + } + } + + let currentUnnamedQueryIndex = 0; + for (const symbol of symbols) { + switch (symbol.type) { + case 'query': + lenses.push( + { + range: symbol.lensRange.toJSON(), + command: { + title: 'Run', + command: 'malloy.runNamedQuery', + arguments: [symbol.name], + }, + }, + { + range: symbol.lensRange.toJSON(), + command: { + title: 'Show SQL', + command: 'malloy.showSQLNamedQuery', + arguments: [symbol.name], + }, + } + ); + break; + case 'unnamed_query': + lenses.push( + { + range: symbol.lensRange.toJSON(), + command: { + title: 'Run', + command: 'malloy.runQueryFile', + arguments: [currentUnnamedQueryIndex], + }, + }, + { + range: symbol.lensRange.toJSON(), + command: { + title: 'Show SQL', + command: 'malloy.showSQLFile', + arguments: [currentUnnamedQueryIndex], + }, + } + ); + currentUnnamedQueryIndex++; + break; + case 'explore': + { + const children = symbol.children; + const exploreName = symbol.name; + lenses.push({ + range: symbol.lensRange.toJSON(), + command: { + title: 'Schema', + command: 'malloy.showSchema', + arguments: [unquoteIdentifier(exploreName)], + }, + }); + lenses.push({ + range: symbol.lensRange.toJSON(), + command: { + title: 'Explore', + command: 'malloy.openComposer', + arguments: [unquoteIdentifier(exploreName)], + }, + }); + if (!externalPreview) { + lenses.push({ + range: symbol.lensRange.toJSON(), + command: { + title: 'Preview', + command: 'malloy.runQuery', + arguments: [ + `run: ${exploreName}->{ select: *; limit: 20 }`, + `Preview: ${exploreName}`, + 'preview', + ], + }, + }); + } + children.forEach(child => { + if (child.type === 'query') { + const queryName = child.name; + lenses.push( + { + range: child.lensRange.toJSON(), + command: { + title: 'Run', + command: 'malloy.runQuery', + arguments: [ + `run: ${exploreName}->${queryName}`, + `${exploreName}->${queryName}`, + ], + }, + }, + { + range: child.lensRange.toJSON(), + command: { + title: 'Show SQL', + command: 'malloy.showSQL', + arguments: [ + `run: ${exploreName}->${queryName}`, + `${exploreName}->${queryName}`, + ], + }, + }, + { + range: child.lensRange.toJSON(), + command: { + title: 'Explore', + command: 'malloy.openComposer', + arguments: [unquoteIdentifier(exploreName), queryName], + }, + } + ); + } + }); + } + break; + case 'import': + try { + const documentUrl = new URL(document.uri); + const url = await fixNotebookUrl( + connection, + new URL(symbol.name, documentUrl) + ); + lenses.push({ + range: symbol.lensRange.toJSON(), + command: { + title: 'Schemas: all', + command: 'malloy.showSchemaFile', + arguments: [url.toString()], + }, + }); + for (const child of symbol.children) { + lenses.push({ + range: child.lensRange.toJSON(), + command: { + title: child.name, + command: 'malloy.showSchema', + arguments: [child.name], + }, + }); + } + symbol.children.forEach((child, idx) => { + const documentMeta: DocumentMetadata = { + uri: url.toString(), + fileName: url.pathname, + languageId: 'malloy', + version: 0, + }; + lenses.push({ + range: child.lensRange.toJSON(), + command: { + title: idx === 0 ? `Explore: ${child.name}` : child.name, + command: 'malloy.openComposer', + arguments: [ + unquoteIdentifier(child.name), + undefined, + undefined, + documentMeta, + ], + }, + }); + }); + } catch (e) { + console.error('import code lens failed with', e); + } + break; + } + } + + return lenses; +} + +function inRange(range: Range, position: Position): boolean { + const {start, end} = range; + const afterStart = + position.line > start.line || + (position.line === start.line && position.character >= start.character); + const beforeEnd = + position.line < end.line || + (position.line === end.line && position.character <= end.character); + return afterStart && beforeEnd; +} + +export async function findMalloyLensesAt( + connection: Connection, + document: TextDocument, + position: Position, + connectionManager: ConnectionManager +): Promise { + const lenses = await getMalloyLenses(connection, document, connectionManager); + + return lenses.filter(lens => inRange(lens.range, position)); +} diff --git a/packages/malloy-language-server/src/lenses/utils.ts b/packages/malloy-language-server/src/lenses/utils.ts new file mode 100644 index 000000000..f9041fdcf --- /dev/null +++ b/packages/malloy-language-server/src/lenses/utils.ts @@ -0,0 +1,20 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type {Connection} from '@malloydata/malloy'; + +export async function getSourceUrl( + tablePath: string, + connection: Connection +): Promise { + const metadata = await connection.fetchTableMetadata(tablePath); + return metadata.url; +} + +export const unquoteIdentifier = (identifier: string): string => + identifier + .split('.') + .map(part => part.replace(/(^`|`$)/g, '')) + .join('.'); diff --git a/packages/malloy-language-server/src/node/connection_factory.ts b/packages/malloy-language-server/src/node/connection_factory.ts new file mode 100644 index 000000000..8f1ee93ac --- /dev/null +++ b/packages/malloy-language-server/src/node/connection_factory.ts @@ -0,0 +1,16 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import '@malloydata/malloy-connections'; +import '@malloydata/db-publisher'; +import type {ConnectionFactory} from '../common/connections/types'; + +/** + * Node connection factory. The side-effect imports above register backend + * connection types with the malloy registry. No additional methods needed — + * discovery and working-directory injection are now handled by core's + * discoverConfig and contextOverlay. + */ +export class NodeConnectionFactory implements ConnectionFactory {} diff --git a/packages/malloy-language-server/src/node/index.ts b/packages/malloy-language-server/src/node/index.ts new file mode 100644 index 000000000..1b9f29637 --- /dev/null +++ b/packages/malloy-language-server/src/node/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +export {NodeConnectionFactory} from './connection_factory'; diff --git a/packages/malloy-language-server/src/node_url_reader.ts b/packages/malloy-language-server/src/node_url_reader.ts new file mode 100644 index 000000000..d7624bbad --- /dev/null +++ b/packages/malloy-language-server/src/node_url_reader.ts @@ -0,0 +1,27 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type {URLReader} from '@malloydata/malloy'; +import * as fs from 'fs'; +import {fileURLToPath} from 'url'; + +export class NodeURLReader implements URLReader { + async readURL(url: URL): Promise { + if (url.protocol === 'file:') { + const filePath = fileURLToPath(url); + return fs.promises.readFile(filePath, 'utf-8'); + } else if (url.protocol === 'http:' || url.protocol === 'https:') { + const response = await fetch(url.toString()); + if (!response.ok) { + throw new Error( + `Failed to fetch ${url}: ${response.status} ${response.statusText}` + ); + } + return response.text(); + } else { + throw new Error(`Unsupported URL protocol: ${url.protocol}`); + } + } +} diff --git a/packages/malloy-language-server/src/parse_cache.ts b/packages/malloy-language-server/src/parse_cache.ts new file mode 100644 index 000000000..7b90edca5 --- /dev/null +++ b/packages/malloy-language-server/src/parse_cache.ts @@ -0,0 +1,63 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type {TextDocument} from 'vscode-languageserver-textdocument'; +import type {Parse} from '@malloydata/malloy'; +import {Malloy} from '@malloydata/malloy'; +import type {MalloySQLParse, MalloySQLSQLParse} from '@malloydata/malloy-sql'; +import {MalloySQLParser, MalloySQLSQLParser} from '@malloydata/malloy-sql'; + +const PARSE_CACHE = new Map(); +const MALLOYSQL_PARSE_CACHE = new Map< + string, + {parsed: MalloySQLParse; version: number} +>(); +const MALLOYSQLSQL_PARSE_CACHE = new Map< + string, + {parsed: MalloySQLSQLParse; version: number} +>(); + +export const parseWithCache = (document: TextDocument): Parse => { + const {version, uri} = document; + + const entry = PARSE_CACHE.get(uri); + if (entry && entry.version === version) { + return entry.parsed; + } + + const parsed = Malloy.parse({source: document.getText()}); + PARSE_CACHE.set(uri, {parsed, version}); + return parsed; +}; + +export const parseMalloySQLWithCache = ( + document: TextDocument +): MalloySQLParse => { + const {version, uri} = document; + + const entry = MALLOYSQL_PARSE_CACHE.get(uri); + if (entry && entry.version === version) { + return entry.parsed; + } + + const parsed = MalloySQLParser.parse(document.getText(), uri); + MALLOYSQL_PARSE_CACHE.set(uri, {parsed, version}); + return parsed; +}; + +export const parseMalloySQLSQLWithCache = ( + document: TextDocument +): MalloySQLSQLParse => { + const {version, uri} = document; + + const entry = MALLOYSQLSQL_PARSE_CACHE.get(uri); + if (entry && entry.version === version) { + return entry.parsed; + } + + const parsed = MalloySQLSQLParser.parse(document.getText(), uri); + MALLOYSQLSQL_PARSE_CACHE.set(uri, {parsed, version}); + return parsed; +}; diff --git a/packages/malloy-language-server/src/symbols/index.ts b/packages/malloy-language-server/src/symbols/index.ts new file mode 100644 index 000000000..803aa7753 --- /dev/null +++ b/packages/malloy-language-server/src/symbols/index.ts @@ -0,0 +1,6 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +export {getMalloySymbols} from './symbols'; diff --git a/packages/malloy-language-server/src/symbols/symbols.ts b/packages/malloy-language-server/src/symbols/symbols.ts new file mode 100644 index 000000000..349eb5ad2 --- /dev/null +++ b/packages/malloy-language-server/src/symbols/symbols.ts @@ -0,0 +1,49 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type {TextDocument} from 'vscode-languageserver-textdocument'; +import type {DocumentSymbol as MalloyDocumentSymbol} from '@malloydata/malloy'; +import type {DocumentSymbol} from 'vscode-languageserver'; +import {SymbolKind} from 'vscode-languageserver'; +import {parseWithCache} from '../parse_cache'; + +function mapSymbol({ + name, + range, + type, + children, +}: MalloyDocumentSymbol): DocumentSymbol { + let kind: SymbolKind; + let detail = type; + switch (type) { + case 'explore': + kind = SymbolKind.Namespace; + detail = 'source'; + break; + case 'query': + kind = SymbolKind.Class; + break; + case 'join': + kind = SymbolKind.Interface; + break; + case 'unnamed_query': + kind = SymbolKind.Class; + break; + default: + kind = SymbolKind.Field; + } + return { + name: name || 'unnamed', + range: range.toJSON(), + detail, + kind, + selectionRange: range.toJSON(), + children: children.map(mapSymbol), + }; +} + +export function getMalloySymbols(document: TextDocument): DocumentSymbol[] { + return parseWithCache(document).symbols.map(mapSymbol); +} diff --git a/packages/malloy-language-server/src/translate_cache.ts b/packages/malloy-language-server/src/translate_cache.ts new file mode 100644 index 000000000..a5f97e490 --- /dev/null +++ b/packages/malloy-language-server/src/translate_cache.ts @@ -0,0 +1,368 @@ +/* + * Copyright Contributors to the Malloy project + * SPDX-License-Identifier: MIT + */ + +import type {Connection, TextDocuments} from 'vscode-languageserver'; +import type { + Model, + ModelMaterializer, + URLReader, + CachedModel, +} from '@malloydata/malloy'; +import {CacheManager, MalloyError, Runtime} from '@malloydata/malloy'; +import type {TextDocument} from 'vscode-languageserver-textdocument'; + +import type {ConnectionManager} from './common/types/connection_manager_types'; +import type {BuildModelRequest, CellData} from './common/types/file_handler'; +import {MalloySQLSQLParser} from '@malloydata/malloy-sql'; +import {fixLogRange} from './common/malloy_sql'; +import {prettyTime, prettyLogUri, prettyLogInvalidationKey} from './common/log'; + +/** + * Logger interface so TranslateCache doesn't depend on a Connection for logging. + */ +export interface TranslateCacheLogger { + info(message: string): void; + debug(message: string): void; + error(message: string): void; +} + +/** + * Optional interface for fetching notebook cell data. + * Only needed in VS Code — standalone mode throws if notebook cells are encountered. + */ +export interface CellDataProvider { + fetchCellData(uri: string): Promise; +} + +/** + * Optional interface for resolving workspace folders. + * Only needed for untitled: document URI handling in VS Code. + */ +export interface WorkspaceFolderProvider { + getWorkspaceFolders(): Promise<{uri: string}[] | null>; +} + +/** + * Inbound request handler type — called when a client sends malloy/fetchModel. + */ +export interface FetchModelResult { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + explores: any[]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + queries: any[]; +} + +export class TranslateCache { + private readonly truncatedCache = new Map< + string, + {model: Model; exploreCount: number; version: number} + >(); + + private readonly cache = new Map(); + + private cacheManager = new CacheManager(this); + + public deleteModel(uri: string) { + this.cache.delete(uri); + this.truncatedCache.delete(uri); + } + + public deleteAllModels() { + this.cache.clear(); + this.truncatedCache.clear(); + } + + public async getModel(url: URL): Promise { + const _url = url.toString(); + const result = this.cache.get(_url); + const prettyUri = prettyLogUri(_url); + this.logger.info( + `translateWithCache ${prettyUri} ${result ? 'hit' : 'miss'}` + ); + return Promise.resolve(result); + } + + public async setModel(url: URL, cachedModel: CachedModel): Promise { + const _url = url.toString(); + const prettyUri = prettyLogUri(_url); + this.logger.info( + `translateWithCache ${prettyUri} ${prettyLogInvalidationKey( + cachedModel.invalidationKeys[_url] + )} set` + ); + this.cache.set(_url, cachedModel); + return Promise.resolve(true); + } + + public dependenciesFor(uri: string): string[] | undefined { + const entry = this.cache.get(uri); + if (entry === undefined) return undefined; + return Object.keys(entry.invalidationKeys ?? {}).filter( + other => other !== uri + ); + } + + public dependentsOf(uri: string): string[] | undefined { + if (!this.cache.has(uri)) return undefined; + const dependencies: string[] = []; + const [base, hash] = uri.split('#'); + for (const [otherURI, model] of this.cache.entries()) { + if (otherURI === uri) continue; + if (Object.keys(model.invalidationKeys).includes(uri)) { + dependencies.push(otherURI); + } + if (otherURI.startsWith(base)) { + const [_, keyHash] = otherURI.split('#'); + if (keyHash > hash) { + dependencies.push(otherURI); + } + } + } + return dependencies; + } + + constructor( + private documents: TextDocuments, + private logger: TranslateCacheLogger, + private connectionManager: ConnectionManager, + private urlReader: URLReader, + private cellDataProvider?: CellDataProvider, + private workspaceFolderProvider?: WorkspaceFolderProvider, + connection?: Connection + ) { + // Register the inbound malloy/fetchModel handler if a Connection is provided + if (connection) { + connection.onRequest( + 'malloy/fetchModel', + async (event: BuildModelRequest): Promise => { + const model = await this.translateWithCache( + event.uri, + event.languageId, + event.refreshSchemaCache + ); + if (model) { + return { + explores: model.explores.map(explore => explore.toJSON()) || [], + queries: model.namedQueries, + }; + } else { + return { + explores: [], + queries: [], + }; + } + } + ); + } + } + + async getDocumentText( + documents: TextDocuments, + uri: URL + ): Promise { + const cached = documents.get(uri.toString()); + if (cached) { + return cached.getText(); + } else { + this.logger.info('fetchFile requesting ' + uri.toString()); + const result = await this.urlReader.readURL(uri); + return typeof result === 'string' ? result : result.contents; + } + } + + async createModelMaterializer( + uri: string, + runtime: Runtime, + refreshSchemaCache?: boolean | number + ): Promise { + const prettyUri = prettyLogUri(uri); + this.logger.debug(`createModelMaterializer ${prettyUri} start`); + let modelMaterializer: ModelMaterializer | null = null; + const queryFileURL = new URL(uri); + if (queryFileURL.protocol === 'vscode-notebook-cell:') { + if (!this.cellDataProvider) { + throw new Error( + 'Notebook cells (vscode-notebook-cell: URIs) are not supported in standalone mode' + ); + } + if (refreshSchemaCache && typeof refreshSchemaCache !== 'number') { + refreshSchemaCache = Date.now(); + } + const cellData = await this.getCellData(new URL(uri)); + const importBaseURL = new URL(cellData.baseUri); + for (const cell of cellData.cells) { + if (cell.languageId === 'malloy') { + const url = new URL(cell.uri); + if (modelMaterializer) { + modelMaterializer = modelMaterializer.extendModel(url, { + importBaseURL, + refreshSchemaCache, + noThrowOnError: true, + }); + } else { + modelMaterializer = runtime.loadModel(url, { + importBaseURL, + refreshSchemaCache, + noThrowOnError: true, + }); + } + } + } + } else { + let importBaseURL: URL | undefined; + if ( + queryFileURL.protocol === 'untitled:' && + this.workspaceFolderProvider + ) { + const workspaceFolders = + await this.workspaceFolderProvider.getWorkspaceFolders(); + if (workspaceFolders?.[0]) { + importBaseURL = new URL(workspaceFolders[0].uri + '/'); + } + } + modelMaterializer = runtime.loadModel(queryFileURL, { + importBaseURL, + refreshSchemaCache, + noThrowOnError: true, + }); + } + this.logger.debug(`createModelMaterializer ${prettyUri} end`); + return modelMaterializer; + } + + async getCellData(uri: URL): Promise { + if (!this.cellDataProvider) { + throw new Error( + 'Notebook cells (vscode-notebook-cell: URIs) are not supported in standalone mode' + ); + } + return await this.cellDataProvider.fetchCellData(uri.toString()); + } + + private async makeRuntime( + fileURL: URL, + urlReader: {readURL: (url: URL) => Promise} + ): Promise { + const config = await this.connectionManager.getConfigForFile(fileURL); + return new Runtime({ + urlReader, + config, + cacheManager: this.cacheManager, + }); + } + + async translateWithTruncatedCache( + document: TextDocument, + text: string, + exploreCount: number + ): Promise { + const prettyUri = prettyLogUri(document.uri); + this.logger.info(`translateWithTruncatedCache ${prettyUri} start`); + const {uri, languageId} = document; + if (languageId === 'malloy') { + const entry = this.truncatedCache.get(uri); + if ( + entry && + entry.exploreCount === exploreCount && + entry.version === document.version + ) { + this.logger.info(`translateWithTruncatedCache ${prettyUri} hit`); + return entry.model; + } + const urlReader = { + readURL: (url: URL) => { + if (url.toString() === uri) { + return Promise.resolve(text); + } else { + return this.getDocumentText(this.documents, url); + } + }, + }; + const fileURL = new URL(uri); + const runtime = await this.makeRuntime(fileURL, urlReader); + const modelMaterializer = await this.createModelMaterializer( + uri, + runtime, + false + ); + const model = await modelMaterializer?.getModel(); + if (model) { + this.truncatedCache.set(uri, { + model, + exploreCount, + version: document.version, + }); + } + this.logger.info(`translateWithTruncatedCache ${prettyUri} miss`); + return model; + } + return undefined; + } + + async translateWithCache( + uri: string, + languageId: string, + refreshSchemaCache?: boolean + ): Promise { + const prettyUri = prettyLogUri(uri); + const t0 = performance.now(); + this.logger.info(`translateWithCache ${prettyUri} start`); + const urlReader = { + readURL: (url: URL) => this.getDocumentText(this.documents, url), + }; + const fileURL = new URL(uri); + const text = await urlReader.readURL(fileURL); + if (languageId === 'malloy-sql') { + const parse = MalloySQLSQLParser.parse(text, uri); + const runtime = await this.makeRuntime(fileURL, urlReader); + + const modelMaterializer = await this.createModelMaterializer( + uri, + runtime, + refreshSchemaCache + ); + + for (const malloyQuery of parse.embeddedMalloyQueries) { + if (!modelMaterializer) { + throw new Error('Missing model definition'); + } + try { + await modelMaterializer.getQuery(`run:\n${malloyQuery.query}`); + } catch (e) { + if (e instanceof MalloyError) { + e.problems.forEach(log => { + fixLogRange(uri, malloyQuery, log, -1); + }); + } + + throw e; + } + } + + const model = await modelMaterializer?.getModel(); + this.logger.info( + `translateWithCache ${prettyUri} end in ${prettyTime( + performance.now() - t0 + )}s` + ); + return model; + } else { + const runtime = await this.makeRuntime(fileURL, urlReader); + + const modelMaterializer = await this.createModelMaterializer( + uri, + runtime, + refreshSchemaCache + ); + const model = await modelMaterializer?.getModel(); + this.logger.info( + `translateWithCache ${prettyUri} end in ${prettyTime( + performance.now() - t0 + )}` + ); + return model; + } + } +} diff --git a/packages/malloy-language-server/tsconfig.json b/packages/malloy-language-server/tsconfig.json new file mode 100644 index 000000000..619258659 --- /dev/null +++ b/packages/malloy-language-server/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.packages.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "composite": true + }, + "references": [ + {"path": "../malloy"}, + {"path": "../malloy-malloy-sql"}, + {"path": "../malloy-interfaces"}, + {"path": "../malloy-render-validator"} + ], + "include": ["src/**/*.ts"] +}