From 6026bbf5fff4486073d9cefb8f21f05182014eae Mon Sep 17 00:00:00 2001 From: Marco Ippolito Date: Wed, 18 Mar 2026 15:52:10 +0100 Subject: [PATCH 1/2] build: add ts support in core modules --- configure.py | 3 +- node.gyp | 15 ++ src/node_builtins.cc | 10 +- tools/js2c.cc | 103 +++++++-- tools/typescript_transpiler.cc | 412 +++++++++++++++++++++++++++++++++ tools/typescript_transpiler.h | 34 +++ 6 files changed, 550 insertions(+), 27 deletions(-) create mode 100644 tools/typescript_transpiler.cc create mode 100644 tools/typescript_transpiler.h diff --git a/configure.py b/configure.py index fa47e9c48547f2..a7ecddf9cd8c89 100755 --- a/configure.py +++ b/configure.py @@ -1737,7 +1737,8 @@ def gcc_version_ge(version_checked): return True def configure_node_lib_files(o): - o['variables']['node_library_files'] = SearchFiles('lib', 'js') + o['variables']['node_library_files'] = SearchFiles('lib', 'js') + \ + SearchFiles('lib', 'ts') def configure_node_cctest_sources(o): o['variables']['node_cctest_sources'] = [ 'src/node_snapshot_stub.cc' ] + \ diff --git a/node.gyp b/node.gyp index 1366e1a0cd3505..afa4f422072998 100644 --- a/node.gyp +++ b/node.gyp @@ -1552,12 +1552,27 @@ 'target_name': 'node_js2c', 'type': 'executable', 'toolsets': ['host'], + 'dependencies': [ + 'tools/v8_gypfiles/v8.gyp:v8_base_without_compiler', + 'tools/v8_gypfiles/v8.gyp:v8_compiler_for_mksnapshot', + 'tools/v8_gypfiles/v8.gyp:v8_init', + 'tools/v8_gypfiles/v8.gyp:v8_libbase', + 'tools/v8_gypfiles/v8.gyp:v8_libplatform', + 'tools/v8_gypfiles/v8.gyp:v8_maybe_icu', + 'tools/v8_gypfiles/v8.gyp:v8_snapshot', + 'tools/v8_gypfiles/v8.gyp:fp16', + 'tools/v8_gypfiles/abseil.gyp:abseil', + ], 'include_dirs': [ + 'deps/v8', + 'deps/v8/include', 'tools', 'src', ], 'sources': [ 'tools/js2c.cc', + 'tools/typescript_transpiler.cc', + 'tools/typescript_transpiler.h', 'tools/executable_wrapper.h', 'src/embedded_data.h', 'src/embedded_data.cc', diff --git a/src/node_builtins.cc b/src/node_builtins.cc index 318ff5158e9c28..2d2a6a76c3eeb6 100644 --- a/src/node_builtins.cc +++ b/src/node_builtins.cc @@ -188,7 +188,15 @@ static std::string OnDiskFileName(const char* id) { // V8 tools scripts are .mjs files. filename += ".mjs"; } else { - filename += ".js"; + std::string ts_filename = filename + ".ts"; + uv_fs_t req; + int err = uv_fs_access(nullptr, &req, ts_filename.c_str(), 0, nullptr); + uv_fs_req_cleanup(&req); + if (err == 0) { + filename = std::move(ts_filename); + } else { + filename += ".js"; + } } return filename; diff --git a/tools/js2c.cc b/tools/js2c.cc index 2cb09f8e1d7ba6..c6515541e53ae9 100644 --- a/tools/js2c.cc +++ b/tools/js2c.cc @@ -13,6 +13,7 @@ #include "embedded_data.h" #include "executable_wrapper.h" #include "simdutf.h" +#include "typescript_transpiler.h" #include "uv.h" #if defined(_WIN32) @@ -131,13 +132,14 @@ bool SearchFiles(const std::string& dir, constexpr std::string_view kMjsSuffix = ".mjs"; constexpr std::string_view kJsSuffix = ".js"; +constexpr std::string_view kTsSuffix = ".ts"; constexpr std::string_view kGypiSuffix = ".gypi"; constexpr std::string_view depsPrefix = "deps/"; constexpr std::string_view libPrefix = "lib/"; constexpr std::string_view HasAllowedExtensions( const std::string_view filename) { - for (const auto& ext : {kGypiSuffix, kJsSuffix, kMjsSuffix}) { + for (const auto& ext : {kGypiSuffix, kJsSuffix, kMjsSuffix, kTsSuffix}) { if (filename.ends_with(ext)) { return ext; } @@ -329,11 +331,13 @@ std::string GetFileId(const std::string& filename) { size_t end = filename.size(); size_t start = 0; std::string prefix; - // Strip .mjs and .js suffix + // Strip .mjs, .js and .ts suffix if (filename.ends_with(kMjsSuffix)) { end -= kMjsSuffix.size(); } else if (filename.ends_with(kJsSuffix)) { end -= kJsSuffix.size(); + } else if (filename.ends_with(kTsSuffix)) { + end -= kTsSuffix.size(); } // deps/acorn/acorn/dist/acorn.js -> internal/deps/acorn/acorn/dist/acorn @@ -670,6 +674,7 @@ Fragment GetDefinition(const std::string& var, const std::vector& code) { } int AddModule(const std::string& filename, + TypeScriptTranspiler* transpiler, Fragments* definitions, Fragments* initializers, Fragments* registrations) { @@ -684,6 +689,21 @@ int AddModule(const std::string& filename, if (error != 0) { return error; } + + if (filename.ends_with(kTsSuffix)) { + std::vector transpiled; + if (transpiler->Strip(std::string_view(code.data(), code.size()), + filename, + &transpiled) != 0) { + fprintf(stderr, + "Failed to transpile TypeScript file %s: %s\n", + filename.c_str(), + std::string(transpiler->LastError()).c_str()); + return 1; + } + code = std::move(transpiled); + } + std::string file_id = GetFileId(filename); std::string var = GetVariableName(file_id); @@ -826,23 +846,39 @@ int AddGypi(const std::string& var, int JS2C(const FileList& js_files, const FileList& mjs_files, + const FileList& ts_files, const std::string& config, + const char* argv0, const std::string& dest) { + TypeScriptTranspiler transpiler; Fragments definitions; - definitions.reserve(js_files.size() + mjs_files.size() + 1); + definitions.reserve(js_files.size() + mjs_files.size() + ts_files.size() + 1); Fragments initializers; - initializers.reserve(js_files.size() + mjs_files.size()); + initializers.reserve(js_files.size() + mjs_files.size() + ts_files.size()); Fragments registrations; - registrations.reserve(js_files.size() + mjs_files.size() + 1); + registrations.reserve(js_files.size() + mjs_files.size() + ts_files.size() + + 1); - for (const auto& filename : js_files) { - int r = AddModule(filename, &definitions, &initializers, ®istrations); - if (r != 0) { - return r; - } + if (!ts_files.empty() && transpiler.Initialize(argv0) != 0) { + fprintf(stderr, + "Failed to initialize TypeScript transpiler: %s\n", + std::string(transpiler.LastError()).c_str()); + return 1; } - for (const auto& filename : mjs_files) { - int r = AddModule(filename, &definitions, &initializers, ®istrations); + + auto add_modules = [&](const FileList& files) { + for (const auto& filename : files) { + int r = AddModule( + filename, &transpiler, &definitions, &initializers, ®istrations); + if (r != 0) { + return r; + } + } + return 0; + }; + + for (const auto* files : {&js_files, &mjs_files, &ts_files}) { + int r = add_modules(*files); if (r != 0) { return r; } @@ -910,7 +946,8 @@ int Main(int argc, char* argv[]) { const std::string& file = args[i]; if (IsDirectory(file, &error)) { if (!SearchFiles(file, &file_map, kJsSuffix) || - !SearchFiles(file, &file_map, kMjsSuffix)) { + !SearchFiles(file, &file_map, kMjsSuffix) || + !SearchFiles(file, &file_map, kTsSuffix)) { return 1; } } else if (error != 0) { @@ -927,8 +964,6 @@ int Main(int argc, char* argv[]) { } } - // Should have exactly 3 types: `.js`, `.mjs` and `.gypi`. - assert(file_map.size() == 3); auto gypi_it = file_map.find(".gypi"); // Currently config.gypi is the only `.gypi` file allowed if (gypi_it == file_map.end() || gypi_it->second.size() != 1 || @@ -940,19 +975,37 @@ int Main(int argc, char* argv[]) { } auto js_it = file_map.find(".js"); auto mjs_it = file_map.find(".mjs"); - assert(js_it != file_map.end() && mjs_it != file_map.end()); - - auto it = std::find(mjs_it->second.begin(), - mjs_it->second.end(), - "lib/eslint.config_partial.mjs"); - if (it != mjs_it->second.end()) { - mjs_it->second.erase(it); + auto ts_it = file_map.find(".ts"); + if (mjs_it != file_map.end()) { + auto it = std::find(mjs_it->second.begin(), + mjs_it->second.end(), + "lib/eslint.config_partial.mjs"); + if (it != mjs_it->second.end()) { + mjs_it->second.erase(it); + } } - std::sort(js_it->second.begin(), js_it->second.end()); - std::sort(mjs_it->second.begin(), mjs_it->second.end()); + if (js_it != file_map.end()) { + std::sort(js_it->second.begin(), js_it->second.end()); + } + if (mjs_it != file_map.end()) { + std::sort(mjs_it->second.begin(), mjs_it->second.end()); + } + if (ts_it != file_map.end()) { + std::sort(ts_it->second.begin(), ts_it->second.end()); + } - return JS2C(js_it->second, mjs_it->second, gypi_it->second[0], output); + static const FileList empty_list; + const FileList& js_files = + js_it == file_map.end() ? empty_list : js_it->second; + const FileList& mjs_files = + mjs_it == file_map.end() ? empty_list : mjs_it->second; + return JS2C(js_files, + mjs_files, + ts_it == file_map.end() ? empty_list : ts_it->second, + gypi_it->second[0], + argv[0], + output); } } // namespace js2c } // namespace node diff --git a/tools/typescript_transpiler.cc b/tools/typescript_transpiler.cc new file mode 100644 index 00000000000000..d138605861489a --- /dev/null +++ b/tools/typescript_transpiler.cc @@ -0,0 +1,412 @@ +#include "typescript_transpiler.h" + +#include +#include +#include +#include + +#include "libplatform/libplatform.h" +#include "uv.h" +#include "v8.h" + +namespace node { +namespace js2c { + +namespace { + +using v8::ArrayBuffer; +using v8::Context; +using v8::Function; +using v8::Global; +using v8::HandleScope; +using v8::Isolate; +using v8::Local; +using v8::Message; +using v8::MaybeLocal; +using v8::Object; +using v8::Platform; +using v8::Script; +using v8::ScriptCompiler; +using v8::ScriptOrigin; +using v8::String; +using v8::TryCatch; +using v8::V8; +using v8::Value; + +constexpr std::string_view kAmaroPath = "deps/amaro/dist/index.js"; + +constexpr char kBootstrapScript[] = R"JS((function(globalThis) { + function encodeUtf8(input) { + return Uint8Array.from( + unescape(encodeURIComponent(String(input))), + (char) => char.charCodeAt(0)); + } + + function decodeUtf8(input) { + let binary = ''; + const chunkSize = 0x8000; + for (let i = 0; i < input.length; i += chunkSize) { + binary += String.fromCharCode.apply( + String, + input.subarray(i, i + chunkSize)); + } + return decodeURIComponent(escape(binary)); + } + + class TextEncoder { + encode(input = '') { + return encodeUtf8(input); + } + } + + class TextDecoder { + constructor(encoding = 'utf-8', options = {}) { + if (encoding !== 'utf-8' || !options.ignoreBOM) { + throw new Error('Unexpected TextDecoder usage'); + } + } + + decode(input = new Uint8Array(), options = undefined) { + if (!(input instanceof Uint8Array) || options !== undefined) { + throw new Error('Unexpected TextDecoder usage'); + } + return decodeUtf8(input); + } + } + + const Buffer = { + from(value, encoding) { + if (encoding === 'base64') { + return Uint8Array.fromBase64(String(value)); + } + throw new Error(`Unsupported Buffer.from encoding: ${encoding}`); + } + }; + + globalThis.require = (id) => { + if (id === 'util') { + return { TextDecoder, TextEncoder }; + } + if (id === 'node:buffer') { + return { Buffer }; + } + throw new Error(`Unsupported require: ${id}`); + }; + + globalThis.module = { exports: {} }; +})(globalThis); +)JS"; + +MaybeLocal ReadUtf8String(Isolate* isolate, std::string_view source) { + return String::NewFromUtf8(isolate, + source.data(), + v8::NewStringType::kNormal, + static_cast(source.size())); +} + +std::string ToUtf8(Isolate* isolate, Local value) { + String::Utf8Value string(isolate, value); + if (*string == nullptr) { + return {}; + } + return std::string(*string, string.length()); +} + +std::string FormatException(Isolate* isolate, TryCatch* try_catch) { + std::string message = ToUtf8(isolate, try_catch->Exception()); + Local exception_message = try_catch->Message(); + if (exception_message.IsEmpty()) { + return message.empty() ? "Unknown exception" : message; + } + std::string location = ToUtf8(isolate, exception_message->Get()); + if (!location.empty()) { + return location; + } + return message.empty() ? "Unknown exception" : message; +} + +int CloseFile(uv_file file) { + uv_fs_t req; + int err = uv_fs_close(nullptr, &req, file, nullptr); + uv_fs_req_cleanup(&req); + return err; +} + +} // namespace + +class TypeScriptTranspiler::Impl { + public: + Impl() = default; + + ~Impl() { + transform_sync_.Reset(); + context_.Reset(); + if (isolate_ != nullptr) { + isolate_->Dispose(); + } + if (v8_initialized_) { + v8::V8::Dispose(); + } + if (platform_initialized_) { + v8::V8::DisposePlatform(); + } + } + + int Initialize(const char* argv0) { + if (isolate_ != nullptr) { + return 0; + } + + last_error_.clear(); + + V8::InitializeICUDefaultLocation(argv0); + V8::InitializeExternalStartupData(argv0); + platform_ = v8::platform::NewDefaultPlatform(); + V8::InitializePlatform(platform_.get()); + platform_initialized_ = true; + V8::Initialize(); + v8_initialized_ = true; + + allocator_.reset(ArrayBuffer::Allocator::NewDefaultAllocator()); + Isolate::CreateParams create_params; + create_params.array_buffer_allocator = allocator_.get(); + isolate_ = Isolate::New(create_params); + + Isolate::Scope isolate_scope(isolate_); + HandleScope handle_scope(isolate_); + Local context = Context::New(isolate_); + context_.Reset(isolate_, context); + Context::Scope context_scope(context); + + int r = RunScript("js2c-amaro-bootstrap", kBootstrapScript); + if (r != 0) { + return r; + } + + std::string source; + if (!ReadFile(kAmaroPath, &source)) { + last_error_ = "Failed to read " + std::string(kAmaroPath); + return 1; + } + r = RunScript(kAmaroPath, source); + if (r != 0) { + return r; + } + + Local global = context->Global(); + Local module_value; + if (!global->Get(context, ReadOnlyString("module")).ToLocal(&module_value) || + !module_value->IsObject()) { + last_error_ = "Failed to load amaro module"; + return 1; + } + + Local module_object = module_value.As(); + Local exports_value; + if (!module_object->Get(context, ReadOnlyString("exports")).ToLocal(&exports_value) || + !exports_value->IsObject()) { + last_error_ = "Failed to access amaro exports"; + return 1; + } + + Local exports_object = exports_value.As(); + Local transform_value; + if (!exports_object->Get(context, ReadOnlyString("transformSync")).ToLocal(&transform_value) || + !transform_value->IsFunction()) { + last_error_ = "Failed to find amaro transformSync"; + return 1; + } + + transform_sync_.Reset(isolate_, transform_value.As()); + return 0; + } + + int Strip(std::string_view source, + std::string_view filename, + std::vector* output) { + if (isolate_ == nullptr) { + last_error_ = "TypeScript transpiler is not initialized"; + return 1; + } + + last_error_.clear(); + + Isolate::Scope isolate_scope(isolate_); + HandleScope handle_scope(isolate_); + Local context = context_.Get(isolate_); + Context::Scope context_scope(context); + TryCatch try_catch(isolate_); + + Local transform = transform_sync_.Get(isolate_); + Local options = Object::New(isolate_); + if (options->Set(context, + ReadOnlyString("mode"), + ReadOnlyString("strip-only")).IsNothing() || + options->Set(context, + ReadOnlyString("filename"), + StringValue(filename)).IsNothing()) { + last_error_ = "Failed to create amaro options"; + return 1; + } + + Local argv[] = { StringValue(source), options }; + Local result; + if (!transform->Call(context, context->Global(), 2, argv).ToLocal(&result)) { + last_error_ = FormatException(isolate_, &try_catch); + return 1; + } + + if (!result->IsObject()) { + last_error_ = "amaro transformSync returned a non-object result"; + return 1; + } + + Local result_object = result.As(); + Local code_value; + if (!result_object->Get(context, ReadOnlyString("code")).ToLocal(&code_value) || + !code_value->IsString()) { + last_error_ = "amaro transformSync returned no code"; + return 1; + } + + std::string stripped = ToUtf8(isolate_, code_value); + output->assign(stripped.begin(), stripped.end()); + return 0; + } + + std::string_view last_error() const { + return last_error_; + } + + private: + bool ReadFile(std::string_view path, std::string* out) { + std::string path_string(path); + uv_fs_t req; + uv_file file = uv_fs_open(nullptr, &req, path_string.c_str(), O_RDONLY, 0, + nullptr); + if (file < 0) { + uv_fs_req_cleanup(&req); + return false; + } + uv_fs_req_cleanup(&req); + + int err = uv_fs_fstat(nullptr, &req, file, nullptr); + if (err < 0) { + uv_fs_req_cleanup(&req); + CloseFile(file); + return false; + } + if (req.statbuf.st_size < 0) { + uv_fs_req_cleanup(&req); + CloseFile(file); + return false; + } + size_t size = static_cast(req.statbuf.st_size); + uv_fs_req_cleanup(&req); + + out->assign(size, '\0'); + if (out->empty()) { + out->clear(); + return CloseFile(file) >= 0; + } + + size_t offset = 0; + while (offset < out->size()) { + uv_buf_t buf = uv_buf_init(out->data() + offset, out->size() - offset); + err = uv_fs_read(nullptr, &req, file, &buf, 1, offset, nullptr); + uv_fs_req_cleanup(&req); + if (err <= 0) { + CloseFile(file); + return false; + } + offset += static_cast(err); + } + + err = uv_fs_close(nullptr, &req, file, nullptr); + uv_fs_req_cleanup(&req); + if (err < 0) { + return false; + } + + return true; + } + + int RunScript(std::string_view name, std::string_view source) { + HandleScope handle_scope(isolate_); + Local context = context_.Get(isolate_); + Context::Scope context_scope(context); + TryCatch try_catch(isolate_); + + Local source_value; + Local name_value; + if (!ReadUtf8String(isolate_, source).ToLocal(&source_value) || + !ReadUtf8String(isolate_, name).ToLocal(&name_value)) { + last_error_ = "Failed to create V8 strings for " + std::string(name); + return 1; + } + + ScriptOrigin origin(name_value); + ScriptCompiler::Source script_source(source_value, origin); + Local