From 8f2bdff7b9f948141dfb00a337f9c2acec6b118e Mon Sep 17 00:00:00 2001 From: whitequark Date: Thu, 26 Sep 2019 02:11:22 +0000 Subject: libs: import json11. This commit imports the code from upstream commit dropbox/json11@8ccf1f0c5ecab6151a65f216e7eeccd8588e5457. --- Makefile | 3 + libs/json11/json11.cpp | 788 +++++++++++++++++++++++++++++++++++++++++++++++++ libs/json11/json11.hpp | 232 +++++++++++++++ 3 files changed, 1023 insertions(+) create mode 100644 libs/json11/json11.cpp create mode 100644 libs/json11/json11.hpp diff --git a/Makefile b/Makefile index bd69ce845..2bd6b218f 100644 --- a/Makefile +++ b/Makefile @@ -528,6 +528,7 @@ $(eval $(call add_include_file,kernel/satgen.h)) $(eval $(call add_include_file,libs/ezsat/ezsat.h)) $(eval $(call add_include_file,libs/ezsat/ezminisat.h)) $(eval $(call add_include_file,libs/sha1/sha1.h)) +$(eval $(call add_include_file,libs/json11/json11.hpp)) $(eval $(call add_include_file,passes/fsm/fsmdata.h)) $(eval $(call add_include_file,frontends/ast/ast.h)) $(eval $(call add_include_file,backends/ilang/ilang_backend.h)) @@ -545,6 +546,8 @@ OBJS += libs/sha1/sha1.o ifneq ($(SMALL),1) +OBJS += libs/json11/json11.o + OBJS += libs/subcircuit/subcircuit.o OBJS += libs/ezsat/ezsat.o diff --git a/libs/json11/json11.cpp b/libs/json11/json11.cpp new file mode 100644 index 000000000..549463d71 --- /dev/null +++ b/libs/json11/json11.cpp @@ -0,0 +1,788 @@ +/* Copyright (c) 2013 Dropbox, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "json11.hpp" +#include +#include +#include +#include +#include + +namespace json11 { + +static const int max_depth = 200; + +using std::string; +using std::vector; +using std::map; +using std::make_shared; +using std::initializer_list; +using std::move; + +/* Helper for representing null - just a do-nothing struct, plus comparison + * operators so the helpers in JsonValue work. We can't use nullptr_t because + * it may not be orderable. + */ +struct NullStruct { + bool operator==(NullStruct) const { return true; } + bool operator<(NullStruct) const { return false; } +}; + +/* * * * * * * * * * * * * * * * * * * * + * Serialization + */ + +static void dump(NullStruct, string &out) { + out += "null"; +} + +static void dump(double value, string &out) { + if (std::isfinite(value)) { + char buf[32]; + snprintf(buf, sizeof buf, "%.17g", value); + out += buf; + } else { + out += "null"; + } +} + +static void dump(int value, string &out) { + char buf[32]; + snprintf(buf, sizeof buf, "%d", value); + out += buf; +} + +static void dump(bool value, string &out) { + out += value ? "true" : "false"; +} + +static void dump(const string &value, string &out) { + out += '"'; + for (size_t i = 0; i < value.length(); i++) { + const char ch = value[i]; + if (ch == '\\') { + out += "\\\\"; + } else if (ch == '"') { + out += "\\\""; + } else if (ch == '\b') { + out += "\\b"; + } else if (ch == '\f') { + out += "\\f"; + } else if (ch == '\n') { + out += "\\n"; + } else if (ch == '\r') { + out += "\\r"; + } else if (ch == '\t') { + out += "\\t"; + } else if (static_cast(ch) <= 0x1f) { + char buf[8]; + snprintf(buf, sizeof buf, "\\u%04x", ch); + out += buf; + } else if (static_cast(ch) == 0xe2 && static_cast(value[i+1]) == 0x80 + && static_cast(value[i+2]) == 0xa8) { + out += "\\u2028"; + i += 2; + } else if (static_cast(ch) == 0xe2 && static_cast(value[i+1]) == 0x80 + && static_cast(value[i+2]) == 0xa9) { + out += "\\u2029"; + i += 2; + } else { + out += ch; + } + } + out += '"'; +} + +static void dump(const Json::array &values, string &out) { + bool first = true; + out += "["; + for (const auto &value : values) { + if (!first) + out += ", "; + value.dump(out); + first = false; + } + out += "]"; +} + +static void dump(const Json::object &values, string &out) { + bool first = true; + out += "{"; + for (const auto &kv : values) { + if (!first) + out += ", "; + dump(kv.first, out); + out += ": "; + kv.second.dump(out); + first = false; + } + out += "}"; +} + +void Json::dump(string &out) const { + m_ptr->dump(out); +} + +/* * * * * * * * * * * * * * * * * * * * + * Value wrappers + */ + +template +class Value : public JsonValue { +protected: + + // Constructors + explicit Value(const T &value) : m_value(value) {} + explicit Value(T &&value) : m_value(move(value)) {} + + // Get type tag + Json::Type type() const override { + return tag; + } + + // Comparisons + bool equals(const JsonValue * other) const override { + return m_value == static_cast *>(other)->m_value; + } + bool less(const JsonValue * other) const override { + return m_value < static_cast *>(other)->m_value; + } + + const T m_value; + void dump(string &out) const override { json11::dump(m_value, out); } +}; + +class JsonDouble final : public Value { + double number_value() const override { return m_value; } + int int_value() const override { return static_cast(m_value); } + bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } + bool less(const JsonValue * other) const override { return m_value < other->number_value(); } +public: + explicit JsonDouble(double value) : Value(value) {} +}; + +class JsonInt final : public Value { + double number_value() const override { return m_value; } + int int_value() const override { return m_value; } + bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } + bool less(const JsonValue * other) const override { return m_value < other->number_value(); } +public: + explicit JsonInt(int value) : Value(value) {} +}; + +class JsonBoolean final : public Value { + bool bool_value() const override { return m_value; } +public: + explicit JsonBoolean(bool value) : Value(value) {} +}; + +class JsonString final : public Value { + const string &string_value() const override { return m_value; } +public: + explicit JsonString(const string &value) : Value(value) {} + explicit JsonString(string &&value) : Value(move(value)) {} +}; + +class JsonArray final : public Value { + const Json::array &array_items() const override { return m_value; } + const Json & operator[](size_t i) const override; +public: + explicit JsonArray(const Json::array &value) : Value(value) {} + explicit JsonArray(Json::array &&value) : Value(move(value)) {} +}; + +class JsonObject final : public Value { + const Json::object &object_items() const override { return m_value; } + const Json & operator[](const string &key) const override; +public: + explicit JsonObject(const Json::object &value) : Value(value) {} + explicit JsonObject(Json::object &&value) : Value(move(value)) {} +}; + +class JsonNull final : public Value { +public: + JsonNull() : Value({}) {} +}; + +/* * * * * * * * * * * * * * * * * * * * + * Static globals - static-init-safe + */ +struct Statics { + const std::shared_ptr null = make_shared(); + const std::shared_ptr t = make_shared(true); + const std::shared_ptr f = make_shared(false); + const string empty_string; + const vector empty_vector; + const map empty_map; + Statics() {} +}; + +static const Statics & statics() { + static const Statics s {}; + return s; +} + +static const Json & static_null() { + // This has to be separate, not in Statics, because Json() accesses statics().null. + static const Json json_null; + return json_null; +} + +/* * * * * * * * * * * * * * * * * * * * + * Constructors + */ + +Json::Json() noexcept : m_ptr(statics().null) {} +Json::Json(std::nullptr_t) noexcept : m_ptr(statics().null) {} +Json::Json(double value) : m_ptr(make_shared(value)) {} +Json::Json(int value) : m_ptr(make_shared(value)) {} +Json::Json(bool value) : m_ptr(value ? statics().t : statics().f) {} +Json::Json(const string &value) : m_ptr(make_shared(value)) {} +Json::Json(string &&value) : m_ptr(make_shared(move(value))) {} +Json::Json(const char * value) : m_ptr(make_shared(value)) {} +Json::Json(const Json::array &values) : m_ptr(make_shared(values)) {} +Json::Json(Json::array &&values) : m_ptr(make_shared(move(values))) {} +Json::Json(const Json::object &values) : m_ptr(make_shared(values)) {} +Json::Json(Json::object &&values) : m_ptr(make_shared(move(values))) {} + +/* * * * * * * * * * * * * * * * * * * * + * Accessors + */ + +Json::Type Json::type() const { return m_ptr->type(); } +double Json::number_value() const { return m_ptr->number_value(); } +int Json::int_value() const { return m_ptr->int_value(); } +bool Json::bool_value() const { return m_ptr->bool_value(); } +const string & Json::string_value() const { return m_ptr->string_value(); } +const vector & Json::array_items() const { return m_ptr->array_items(); } +const map & Json::object_items() const { return m_ptr->object_items(); } +const Json & Json::operator[] (size_t i) const { return (*m_ptr)[i]; } +const Json & Json::operator[] (const string &key) const { return (*m_ptr)[key]; } + +double JsonValue::number_value() const { return 0; } +int JsonValue::int_value() const { return 0; } +bool JsonValue::bool_value() const { return false; } +const string & JsonValue::string_value() const { return statics().empty_string; } +const vector & JsonValue::array_items() const { return statics().empty_vector; } +const map & JsonValue::object_items() const { return statics().empty_map; } +const Json & JsonValue::operator[] (size_t) const { return static_null(); } +const Json & JsonValue::operator[] (const string &) const { return static_null(); } + +const Json & JsonObject::operator[] (const string &key) const { + auto iter = m_value.find(key); + return (iter == m_value.end()) ? static_null() : iter->second; +} +const Json & JsonArray::operator[] (size_t i) const { + if (i >= m_value.size()) return static_null(); + else return m_value[i]; +} + +/* * * * * * * * * * * * * * * * * * * * + * Comparison + */ + +bool Json::operator== (const Json &other) const { + if (m_ptr == other.m_ptr) + return true; + if (m_ptr->type() != other.m_ptr->type()) + return false; + + return m_ptr->equals(other.m_ptr.get()); +} + +bool Json::operator< (const Json &other) const { + if (m_ptr == other.m_ptr) + return false; + if (m_ptr->type() != other.m_ptr->type()) + return m_ptr->type() < other.m_ptr->type(); + + return m_ptr->less(other.m_ptr.get()); +} + +/* * * * * * * * * * * * * * * * * * * * + * Parsing + */ + +/* esc(c) + * + * Format char c suitable for printing in an error message. + */ +static inline string esc(char c) { + char buf[12]; + if (static_cast(c) >= 0x20 && static_cast(c) <= 0x7f) { + snprintf(buf, sizeof buf, "'%c' (%d)", c, c); + } else { + snprintf(buf, sizeof buf, "(%d)", c); + } + return string(buf); +} + +static inline bool in_range(long x, long lower, long upper) { + return (x >= lower && x <= upper); +} + +namespace { +/* JsonParser + * + * Object that tracks all state of an in-progress parse. + */ +struct JsonParser final { + + /* State + */ + const string &str; + size_t i; + string &err; + bool failed; + const JsonParse strategy; + + /* fail(msg, err_ret = Json()) + * + * Mark this parse as failed. + */ + Json fail(string &&msg) { + return fail(move(msg), Json()); + } + + template + T fail(string &&msg, const T err_ret) { + if (!failed) + err = std::move(msg); + failed = true; + return err_ret; + } + + /* consume_whitespace() + * + * Advance until the current character is non-whitespace. + */ + void consume_whitespace() { + while (str[i] == ' ' || str[i] == '\r' || str[i] == '\n' || str[i] == '\t') + i++; + } + + /* consume_comment() + * + * Advance comments (c-style inline and multiline). + */ + bool consume_comment() { + bool comment_found = false; + if (str[i] == '/') { + i++; + if (i == str.size()) + return fail("unexpected end of input after start of comment", false); + if (str[i] == '/') { // inline comment + i++; + // advance until next line, or end of input + while (i < str.size() && str[i] != '\n') { + i++; + } + comment_found = true; + } + else if (str[i] == '*') { // multiline comment + i++; + if (i > str.size()-2) + return fail("unexpected end of input inside multi-line comment", false); + // advance until closing tokens + while (!(str[i] == '*' && str[i+1] == '/')) { + i++; + if (i > str.size()-2) + return fail( + "unexpected end of input inside multi-line comment", false); + } + i += 2; + comment_found = true; + } + else + return fail("malformed comment", false); + } + return comment_found; + } + + /* consume_garbage() + * + * Advance until the current character is non-whitespace and non-comment. + */ + void consume_garbage() { + consume_whitespace(); + if(strategy == JsonParse::COMMENTS) { + bool comment_found = false; + do { + comment_found = consume_comment(); + if (failed) return; + consume_whitespace(); + } + while(comment_found); + } + } + + /* get_next_token() + * + * Return the next non-whitespace character. If the end of the input is reached, + * flag an error and return 0. + */ + char get_next_token() { + consume_garbage(); + if (failed) return static_cast(0); + if (i == str.size()) + return fail("unexpected end of input", static_cast(0)); + + return str[i++]; + } + + /* encode_utf8(pt, out) + * + * Encode pt as UTF-8 and add it to out. + */ + void encode_utf8(long pt, string & out) { + if (pt < 0) + return; + + if (pt < 0x80) { + out += static_cast(pt); + } else if (pt < 0x800) { + out += static_cast((pt >> 6) | 0xC0); + out += static_cast((pt & 0x3F) | 0x80); + } else if (pt < 0x10000) { + out += static_cast((pt >> 12) | 0xE0); + out += static_cast(((pt >> 6) & 0x3F) | 0x80); + out += static_cast((pt & 0x3F) | 0x80); + } else { + out += static_cast((pt >> 18) | 0xF0); + out += static_cast(((pt >> 12) & 0x3F) | 0x80); + out += static_cast(((pt >> 6) & 0x3F) | 0x80); + out += static_cast((pt & 0x3F) | 0x80); + } + } + + /* parse_string() + * + * Parse a string, starting at the current position. + */ + string parse_string() { + string out; + long last_escaped_codepoint = -1; + while (true) { + if (i == str.size()) + return fail("unexpected end of input in string", ""); + + char ch = str[i++]; + + if (ch == '"') { + encode_utf8(last_escaped_codepoint, out); + return out; + } + + if (in_range(ch, 0, 0x1f)) + return fail("unescaped " + esc(ch) + " in string", ""); + + // The usual case: non-escaped characters + if (ch != '\\') { + encode_utf8(last_escaped_codepoint, out); + last_escaped_codepoint = -1; + out += ch; + continue; + } + + // Handle escapes + if (i == str.size()) + return fail("unexpected end of input in string", ""); + + ch = str[i++]; + + if (ch == 'u') { + // Extract 4-byte escape sequence + string esc = str.substr(i, 4); + // Explicitly check length of the substring. The following loop + // relies on std::string returning the terminating NUL when + // accessing str[length]. Checking here reduces brittleness. + if (esc.length() < 4) { + return fail("bad \\u escape: " + esc, ""); + } + for (size_t j = 0; j < 4; j++) { + if (!in_range(esc[j], 'a', 'f') && !in_range(esc[j], 'A', 'F') + && !in_range(esc[j], '0', '9')) + return fail("bad \\u escape: " + esc, ""); + } + + long codepoint = strtol(esc.data(), nullptr, 16); + + // JSON specifies that characters outside the BMP shall be encoded as a pair + // of 4-hex-digit \u escapes encoding their surrogate pair components. Check + // whether we're in the middle of such a beast: the previous codepoint was an + // escaped lead (high) surrogate, and this is a trail (low) surrogate. + if (in_range(last_escaped_codepoint, 0xD800, 0xDBFF) + && in_range(codepoint, 0xDC00, 0xDFFF)) { + // Reassemble the two surrogate pairs into one astral-plane character, per + // the UTF-16 algorithm. + encode_utf8((((last_escaped_codepoint - 0xD800) << 10) + | (codepoint - 0xDC00)) + 0x10000, out); + last_escaped_codepoint = -1; + } else { + encode_utf8(last_escaped_codepoint, out); + last_escaped_codepoint = codepoint; + } + + i += 4; + continue; + } + + encode_utf8(last_escaped_codepoint, out); + last_escaped_codepoint = -1; + + if (ch == 'b') { + out += '\b'; + } else if (ch == 'f') { + out += '\f'; + } else if (ch == 'n') { + out += '\n'; + } else if (ch == 'r') { + out += '\r'; + } else if (ch == 't') { + out += '\t'; + } else if (ch == '"' || ch == '\\' || ch == '/') { + out += ch; + } else { + return fail("invalid escape character " + esc(ch), ""); + } + } + } + + /* parse_number() + * + * Parse a double. + */ + Json parse_number() { + size_t start_pos = i; + + if (str[i] == '-') + i++; + + // Integer part + if (str[i] == '0') { + i++; + if (in_range(str[i], '0', '9')) + return fail("leading 0s not permitted in numbers"); + } else if (in_range(str[i], '1', '9')) { + i++; + while (in_range(str[i], '0', '9')) + i++; + } else { + return fail("invalid " + esc(str[i]) + " in number"); + } + + if (str[i] != '.' && str[i] != 'e' && str[i] != 'E' + && (i - start_pos) <= static_cast(std::numeric_limits::digits10)) { + return std::atoi(str.c_str() + start_pos); + } + + // Decimal part + if (str[i] == '.') { + i++; + if (!in_range(str[i], '0', '9')) + return fail("at least one digit required in fractional part"); + + while (in_range(str[i], '0', '9')) + i++; + } + + // Exponent part + if (str[i] == 'e' || str[i] == 'E') { + i++; + + if (str[i] == '+' || str[i] == '-') + i++; + + if (!in_range(str[i], '0', '9')) + return fail("at least one digit required in exponent"); + + while (in_range(str[i], '0', '9')) + i++; + } + + return std::strtod(str.c_str() + start_pos, nullptr); + } + + /* expect(str, res) + * + * Expect that 'str' starts at the character that was just read. If it does, advance + * the input and return res. If not, flag an error. + */ + Json expect(const string &expected, Json res) { + assert(i != 0); + i--; + if (str.compare(i, expected.length(), expected) == 0) { + i += expected.length(); + return res; + } else { + return fail("parse error: expected " + expected + ", got " + str.substr(i, expected.length())); + } + } + + /* parse_json() + * + * Parse a JSON object. + */ + Json parse_json(int depth) { + if (depth > max_depth) { + return fail("exceeded maximum nesting depth"); + } + + char ch = get_next_token(); + if (failed) + return Json(); + + if (ch == '-' || (ch >= '0' && ch <= '9')) { + i--; + return parse_number(); + } + + if (ch == 't') + return expect("true", true); + + if (ch == 'f') + return expect("false", false); + + if (ch == 'n') + return expect("null", Json()); + + if (ch == '"') + return parse_string(); + + if (ch == '{') { + map data; + ch = get_next_token(); + if (ch == '}') + return data; + + while (1) { + if (ch != '"') + return fail("expected '\"' in object, got " + esc(ch)); + + string key = parse_string(); + if (failed) + return Json(); + + ch = get_next_token(); + if (ch != ':') + return fail("expected ':' in object, got " + esc(ch)); + + data[std::move(key)] = parse_json(depth + 1); + if (failed) + return Json(); + + ch = get_next_token(); + if (ch == '}') + break; + if (ch != ',') + return fail("expected ',' in object, got " + esc(ch)); + + ch = get_next_token(); + } + return data; + } + + if (ch == '[') { + vector data; + ch = get_next_token(); + if (ch == ']') + return data; + + while (1) { + i--; + data.push_back(parse_json(depth + 1)); + if (failed) + return Json(); + + ch = get_next_token(); + if (ch == ']') + break; + if (ch != ',') + return fail("expected ',' in list, got " + esc(ch)); + + ch = get_next_token(); + (void)ch; + } + return data; + } + + return fail("expected value, got " + esc(ch)); + } +}; +}//namespace { + +Json Json::parse(const string &in, string &err, JsonParse strategy) { + JsonParser parser { in, 0, err, false, strategy }; + Json result = parser.parse_json(0); + + // Check for any trailing garbage + parser.consume_garbage(); + if (parser.failed) + return Json(); + if (parser.i != in.size()) + return parser.fail("unexpected trailing " + esc(in[parser.i])); + + return result; +} + +// Documented in json11.hpp +vector Json::parse_multi(const string &in, + std::string::size_type &parser_stop_pos, + string &err, + JsonParse strategy) { + JsonParser parser { in, 0, err, false, strategy }; + parser_stop_pos = 0; + vector json_vec; + while (parser.i != in.size() && !parser.failed) { + json_vec.push_back(parser.parse_json(0)); + if (parser.failed) + break; + + // Check for another object + parser.consume_garbage(); + if (parser.failed) + break; + parser_stop_pos = parser.i; + } + return json_vec; +} + +/* * * * * * * * * * * * * * * * * * * * + * Shape-checking + */ + +bool Json::has_shape(const shape & types, string & err) const { + if (!is_object()) { + err = "expected JSON object, got " + dump(); + return false; + } + + for (auto & item : types) { + if ((*this)[item.first].type() != item.second) { + err = "bad type for " + item.first + " in " + dump(); + return false; + } + } + + return true; +} + +} // namespace json11 diff --git a/libs/json11/json11.hpp b/libs/json11/json11.hpp new file mode 100644 index 000000000..0c47d0509 --- /dev/null +++ b/libs/json11/json11.hpp @@ -0,0 +1,232 @@ +/* json11 + * + * json11 is a tiny JSON library for C++11, providing JSON parsing and serialization. + * + * The core object provided by the library is json11::Json. A Json object represents any JSON + * value: null, bool, number (int or double), string (std::string), array (std::vector), or + * object (std::map). + * + * Json objects act like values: they can be assigned, copied, moved, compared for equality or + * order, etc. There are also helper methods Json::dump, to serialize a Json to a string, and + * Json::parse (static) to parse a std::string as a Json object. + * + * Internally, the various types of Json object are represented by the JsonValue class + * hierarchy. + * + * A note on numbers - JSON specifies the syntax of number formatting but not its semantics, + * so some JSON implementations distinguish between integers and floating-point numbers, while + * some don't. In json11, we choose the latter. Because some JSON implementations (namely + * Javascript itself) treat all numbers as the same type, distinguishing the two leads + * to JSON that will be *silently* changed by a round-trip through those implementations. + * Dangerous! To avoid that risk, json11 stores all numbers as double internally, but also + * provides integer helpers. + * + * Fortunately, double-precision IEEE754 ('double') can precisely store any integer in the + * range +/-2^53, which includes every 'int' on most systems. (Timestamps often use int64 + * or long long to avoid the Y2038K problem; a double storing microseconds since some epoch + * will be exact for +/- 275 years.) + */ + +/* Copyright (c) 2013 Dropbox, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#ifdef _MSC_VER + #if _MSC_VER <= 1800 // VS 2013 + #ifndef noexcept + #define noexcept throw() + #endif + + #ifndef snprintf + #define snprintf _snprintf_s + #endif + #endif +#endif + +namespace json11 { + +enum JsonParse { + STANDARD, COMMENTS +}; + +class JsonValue; + +class Json final { +public: + // Types + enum Type { + NUL, NUMBER, BOOL, STRING, ARRAY, OBJECT + }; + + // Array and object typedefs + typedef std::vector array; + typedef std::map object; + + // Constructors for the various types of JSON value. + Json() noexcept; // NUL + Json(std::nullptr_t) noexcept; // NUL + Json(double value); // NUMBER + Json(int value); // NUMBER + Json(bool value); // BOOL + Json(const std::string &value); // STRING + Json(std::string &&value); // STRING + Json(const char * value); // STRING + Json(const array &values); // ARRAY + Json(array &&values); // ARRAY + Json(const object &values); // OBJECT + Json(object &&values); // OBJECT + + // Implicit constructor: anything with a to_json() function. + template + Json(const T & t) : Json(t.to_json()) {} + + // Implicit constructor: map-like objects (std::map, std::unordered_map, etc) + template ().begin()->first)>::value + && std::is_constructible().begin()->second)>::value, + int>::type = 0> + Json(const M & m) : Json(object(m.begin(), m.end())) {} + + // Implicit constructor: vector-like objects (std::list, std::vector, std::set, etc) + template ().begin())>::value, + int>::type = 0> + Json(const V & v) : Json(array(v.begin(), v.end())) {} + + // This prevents Json(some_pointer) from accidentally producing a bool. Use + // Json(bool(some_pointer)) if that behavior is desired. + Json(void *) = delete; + + // Accessors + Type type() const; + + bool is_null() const { return type() == NUL; } + bool is_number() const { return type() == NUMBER; } + bool is_bool() const { return type() == BOOL; } + bool is_string() const { return type() == STRING; } + bool is_array() const { return type() == ARRAY; } + bool is_object() const { return type() == OBJECT; } + + // Return the enclosed value if this is a number, 0 otherwise. Note that json11 does not + // distinguish between integer and non-integer numbers - number_value() and int_value() + // can both be applied to a NUMBER-typed object. + double number_value() const; + int int_value() const; + + // Return the enclosed value if this is a boolean, false otherwise. + bool bool_value() const; + // Return the enclosed string if this is a string, "" otherwise. + const std::string &string_value() const; + // Return the enclosed std::vector if this is an array, or an empty vector otherwise. + const array &array_items() const; + // Return the enclosed std::map if this is an object, or an empty map otherwise. + const object &object_items() const; + + // Return a reference to arr[i] if this is an array, Json() otherwise. + const Json & operator[](size_t i) const; + // Return a reference to obj[key] if this is an object, Json() otherwise. + const Json & operator[](const std::string &key) const; + + // Serialize. + void dump(std::string &out) const; + std::string dump() const { + std::string out; + dump(out); + return out; + } + + // Parse. If parse fails, return Json() and assign an error message to err. + static Json parse(const std::string & in, + std::string & err, + JsonParse strategy = JsonParse::STANDARD); + static Json parse(const char * in, + std::string & err, + JsonParse strategy = JsonParse::STANDARD) { + if (in) { + return parse(std::string(in), err, strategy); + } else { + err = "null input"; + return nullptr; + } + } + // Parse multiple objects, concatenated or separated by whitespace + static std::vector parse_multi( + const std::string & in, + std::string::size_type & parser_stop_pos, + std::string & err, + JsonParse strategy = JsonParse::STANDARD); + + static inline std::vector parse_multi( + const std::string & in, + std::string & err, + JsonParse strategy = JsonParse::STANDARD) { + std::string::size_type parser_stop_pos; + return parse_multi(in, parser_stop_pos, err, strategy); + } + + bool operator== (const Json &rhs) const; + bool operator< (const Json &rhs) const; + bool operator!= (const Json &rhs) const { return !(*this == rhs); } + bool operator<= (const Json &rhs) const { return !(rhs < *this); } + bool operator> (const Json &rhs) const { return (rhs < *this); } + bool operator>= (const Json &rhs) const { return !(*this < rhs); } + + /* has_shape(types, err) + * + * Return true if this is a JSON object and, for each item in types, has a field of + * the given type. If not, return false and set err to a descriptive message. + */ + typedef std::initializer_list> shape; + bool has_shape(const shape & types, std::string & err) const; + +private: + std::shared_ptr m_ptr; +}; + +// Internal class hierarchy - JsonValue objects are not exposed to users of this API. +class JsonValue { +protected: + friend class Json; + friend class JsonInt; + friend class JsonDouble; + virtual Json::Type type() const = 0; + virtual bool equals(const JsonValue * other) const = 0; + virtual bool less(const JsonValue * other) const = 0; + virtual void dump(std::string &out) const = 0; + virtual double number_value() const; + virtual int int_value() const; + virtual bool bool_value() const; + virtual const std::string &string_value() const; + virtual const Json::array &array_items() const; + virtual const Json &operator[](size_t i) const; + virtual const Json::object &object_items() const; + virtual const Json &operator[](const std::string &key) const; + virtual ~JsonValue() {} +}; + +} // namespace json11 -- cgit v1.2.3 From 99a7f39084cf4b9cd21e2a1e4f4a842993dfd147 Mon Sep 17 00:00:00 2001 From: whitequark Date: Thu, 26 Sep 2019 03:57:16 +0000 Subject: rpc: new frontend. A new pass, connect_rpc, allows any HDL frontend that can read/write JSON from/to stdin/stdout or an unix socket or a named pipe to participate in elaboration as a first class citizen, such that any other HDL supported by Yosys directly or indirectly can transparently instantiate modules handled by this frontend. Recognizing that many HDL frontends emit Verilog, it allows the RPC frontend to direct Yosys to process the result of instantiation via any built-in Yosys frontend. The resulting RTLIL is then hygienically integrated into the overall design. --- Makefile | 1 + frontends/rpc/Makefile.inc | 2 + frontends/rpc/rpc_frontend.cc | 589 ++++++++++++++++++++++++++++++++++++++++++ tests/rpc/.gitignore | 1 + tests/rpc/design.v | 8 + tests/rpc/exec.ys | 5 + tests/rpc/frontend.py | 126 +++++++++ tests/rpc/run-test.sh | 6 + tests/rpc/unix.ys | 6 + 9 files changed, 744 insertions(+) create mode 100644 frontends/rpc/Makefile.inc create mode 100644 frontends/rpc/rpc_frontend.cc create mode 100644 tests/rpc/.gitignore create mode 100644 tests/rpc/design.v create mode 100644 tests/rpc/exec.ys create mode 100644 tests/rpc/frontend.py create mode 100755 tests/rpc/run-test.sh create mode 100644 tests/rpc/unix.ys diff --git a/Makefile b/Makefile index 2bd6b218f..c36aef83f 100644 --- a/Makefile +++ b/Makefile @@ -713,6 +713,7 @@ test: $(TARGETS) $(EXTRA_TARGETS) +cd tests/aiger && bash run-test.sh $(ABCOPT) +cd tests/arch && bash run-test.sh +cd tests/ice40 && bash run-test.sh $(SEEDOPT) + +cd tests/rpc && bash run-test.sh @echo "" @echo " Passed \"make test\"." @echo "" diff --git a/frontends/rpc/Makefile.inc b/frontends/rpc/Makefile.inc new file mode 100644 index 000000000..9af505098 --- /dev/null +++ b/frontends/rpc/Makefile.inc @@ -0,0 +1,2 @@ + +OBJS += frontends/rpc/rpc_frontend.o diff --git a/frontends/rpc/rpc_frontend.cc b/frontends/rpc/rpc_frontend.cc new file mode 100644 index 000000000..b4b2fa3a2 --- /dev/null +++ b/frontends/rpc/rpc_frontend.cc @@ -0,0 +1,589 @@ +/* + * yosys -- Yosys Open SYnthesis Suite + * + * Copyright (C) 2019 whitequark + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + */ + +// The reason the -path mode of connect_rpc uses byte-oriented and not message-oriented sockets, even though +// it is a message-oriented interface, is that the system can place various limits on the message size, which +// are not always transparent or easy to change. Given that generated HDL code get be extremely large, it is +// unwise to rely on those limits being large enough, and using byte-oriented sockets is guaranteed to work. + +#ifndef _WIN32 +#include +#include +#include +#include +#include +#endif + +#include "libs/json11/json11.hpp" +#include "libs/sha1/sha1.h" +#include "kernel/yosys.h" + +YOSYS_NAMESPACE_BEGIN + +#if defined(_WIN32) +static std::wstring str2wstr(const std::string &in) { + if(in == "") return L""; + std::wstring out; + out.resize(MultiByteToWideChar(/*CodePage=*/CP_UTF8, /*dwFlags=*/0, /*lpMultiByteStr=*/&in[0], /*cbMultiByte=*/(int)in.length(), /*lpWideCharStr=*/NULL, /*cchWideChar=*/0)); + int written = MultiByteToWideChar(/*CodePage=*/CP_UTF8, /*dwFlags=*/0, /*lpMultiByteStr=*/&in[0], /*cbMultiByte=*/(int)in.length(), /*lpWideCharStr=*/&out[0], /*cchWideChar=*/(int)out.length()); + log_assert(written == (int)out.length()); + return out; +} + +static std::string wstr2str(const std::wstring &in) { + if(in == L"") return ""; + std::string out; + out.resize(WideCharToMultiByte(/*CodePage=*/CP_UTF8, /*dwFlags=*/0, /*lpWideCharStr=*/&in[0], /*cchWideChar=*/(int)in.length(), /*lpMultiByteStr=*/NULL, /*cbMultiByte=*/0, /*lpDefaultChar=*/NULL, /*lpUsedDefaultChar=*/NULL)); + int written = WideCharToMultiByte(/*CodePage=*/CP_UTF8, /*dwFlags=*/0, /*lpWideCharStr=*/&in[0], /*cchWideChar=*/(int)in.length(), /*lpMultiByteStr=*/&out[0], /*cbMultiByte=*/(int)out.length(), /*lpDefaultChar=*/NULL, /*lpUsedDefaultChar=*/NULL); + log_assert(written == (int)out.length()); + return out; +} + +static std::string get_last_error_str() { + DWORD last_error = GetLastError(); + LPWSTR out_w; + DWORD size_w = FormatMessageW(/*dwFlags=*/FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_ALLOCATE_BUFFER|FORMAT_MESSAGE_IGNORE_INSERTS, /*lpSource=*/NULL, /*dwMessageId=*/last_error, /*dwLanguageId=*/0, /*lpBuffer=*/(LPWSTR)&out_w, /*nSize=*/0, /*Arguments=*/NULL); + if (size_w == 0) + return std::to_string(last_error); + std::string out = wstr2str(std::wstring(out_w, size_w)); + LocalFree(out_w); + return out; +} +#endif + +using json11::Json; + +struct RpcServer { + std::string name; + + RpcServer(const std::string &name) : name(name) { } + virtual ~RpcServer() { } + + virtual void write(const std::string &data) = 0; + virtual std::string read() = 0; + + Json call(const Json &json_request) { + std::string request; + json_request.dump(request); + request += '\n'; + log_debug("RPC frontend request: %s", request.c_str()); + write(request); + + std::string response = read(); + log_debug("RPC frontend response: %s", response.c_str()); + std::string error; + Json json_response = Json::parse(response, error); + if (json_response.is_null()) + log_cmd_error("parsing JSON failed: %s\n", error.c_str()); + if (json_response["error"].is_string()) + log_cmd_error("RPC frontend returned an error: %s\n", json_response["error"].string_value().c_str()); + return json_response; + } + + std::vector get_module_names() { + Json response = call(Json::object { + { "method", "modules" }, + }); + bool is_valid = true; + std::vector modules; + if (response["modules"].is_array()) { + for (auto &json_module : response["modules"].array_items()) { + if (json_module.is_string()) + modules.push_back(json_module.string_value()); + else is_valid = false; + } + } else is_valid = false; + if (!is_valid) + log_cmd_error("RPC frontend returned malformed response: %s\n", response.dump().c_str()); + return modules; + } + + std::pair derive_module(const std::string &module, const dict ¶meters) { + Json::object json_parameters; + for (auto ¶m : parameters) { + std::string type, value; + if (param.second.flags & RTLIL::CONST_FLAG_REAL) { + type = "real"; + value = param.second.decode_string(); + } else if (param.second.flags & RTLIL::CONST_FLAG_STRING) { + type = "string"; + value = param.second.decode_string(); + } else if ((param.second.flags & ~RTLIL::CONST_FLAG_SIGNED) == RTLIL::CONST_FLAG_NONE) { + type = (param.second.flags & RTLIL::CONST_FLAG_SIGNED) ? "signed" : "unsigned"; + value = param.second.as_string(); + } else + log_cmd_error("Unserializable constant flags 0x%x\n", param.second.flags); + json_parameters[param.first.str()] = Json::object { + { "type", type }, + { "value", value }, + }; + } + Json response = call(Json::object { + { "method", "derive" }, + { "module", module }, + { "parameters", json_parameters }, + }); + bool is_valid = true; + std::string frontend, source; + if (response["frontend"].is_string()) + frontend = response["frontend"].string_value(); + else is_valid = false; + if (response["source"].is_string()) + source = response["source"].string_value(); + else is_valid = false; + if (!is_valid) + log_cmd_error("RPC frontend returned malformed response: %s\n", response.dump().c_str()); + return std::make_pair(frontend, source); + } +}; + +struct RpcModule : RTLIL::Module { + std::shared_ptr server; + + RTLIL::IdString derive(RTLIL::Design *design, dict parameters, bool /*mayfail*/) YS_OVERRIDE { + std::string stripped_name = name.str(); + if (stripped_name.compare(0, 9, "$abstract") == 0) + stripped_name = stripped_name.substr(9); + log_assert(stripped_name[0] == '\\'); + + log_header(design, "Executing RPC frontend `%s' for module `%s'.\n", server->name.c_str(), stripped_name.c_str()); + + std::string parameter_info; + for (auto ¶m : parameters) { + log("Parameter %s = %s\n", param.first.c_str(), log_signal(RTLIL::SigSpec(param.second))); + parameter_info += stringf("%s=%s", param.first.c_str(), log_signal(RTLIL::SigSpec(param.second))); + } + + std::string derived_name; + if (parameters.empty()) + derived_name = stripped_name; + else if (parameter_info.size() > 60) + derived_name = "$paramod$" + sha1(parameter_info) + stripped_name; + else + derived_name = "$paramod" + stripped_name + parameter_info; + + if (design->has(derived_name)) { + log("Found cached RTLIL representation for module `%s'.\n", derived_name.c_str()); + } else { + std::string command, input; + std::tie(command, input) = server->derive_module(stripped_name.substr(1), parameters); + + std::istringstream input_stream(input); + RTLIL::Design *derived_design = new RTLIL::Design; + Frontend::frontend_call(derived_design, &input_stream, "" + derived_name.substr(8), command); + derived_design->check(); + + dict name_mangling; + bool found_derived_top = false; + for (auto module : derived_design->modules()) { + std::string original_name = module->name.str(); + if (original_name == stripped_name) { + found_derived_top = true; + name_mangling[original_name] = derived_name; + } else { + name_mangling[original_name] = derived_name + module->name.str(); + } + } + if (!found_derived_top) + log_cmd_error("RPC frontend did not return requested module `%s`!\n", stripped_name.c_str()); + + for (auto module : derived_design->modules()) + for (auto cell : module->cells()) + if (name_mangling.count(cell->type.str())) + cell->type = name_mangling[cell->type.str()]; + + for (auto module : derived_design->modules_) { + std::string mangled_name = name_mangling[module.first.str()]; + + log("Importing `%s' as `%s'.\n", log_id(module.first), log_id(mangled_name)); + + module.second->name = mangled_name; + module.second->design = design; + module.second->attributes.erase("\\top"); + design->modules_[mangled_name] = module.second; + derived_design->modules_.erase(module.first); + } + + delete derived_design; + } + + return derived_name; + } + + RTLIL::Module *clone() const YS_OVERRIDE { + RpcModule *new_mod = new RpcModule; + new_mod->server = server; + cloneInto(new_mod); + return new_mod; + } +}; + +#if defined(_WIN32) + +struct HandleRpcServer : RpcServer { + HANDLE hsend, hrecv; + + HandleRpcServer(const std::string &name, HANDLE hsend, HANDLE hrecv) + : RpcServer(name), hsend(hsend), hrecv(hrecv) { } + + void write(const std::string &data) YS_OVERRIDE { + log_assert(data.length() >= 1 && data.find('\n') == data.length() - 1); + ssize_t offset = 0; + do { + DWORD data_written; + if (!WriteFile(hsend, &data[offset], data.length() - offset, &data_written, /*lpOverlapped=*/NULL)) + log_cmd_error("WriteFile failed: %s\n", get_last_error_str().c_str()); + offset += data_written; + } while(offset < (ssize_t)data.length()); + } + + std::string read() YS_OVERRIDE { + std::string data; + ssize_t offset = 0; + while (data.length() == 0 || data[data.length() - 1] != '\n') { + data.resize(data.length() + 1024); + DWORD data_read; + if (!ReadFile(hrecv, &data[offset], data.length() - offset, &data_read, /*lpOverlapped=*/NULL)) + log_cmd_error("ReadFile failed: %s\n", get_last_error_str().c_str()); + offset += data_read; + data.resize(offset); + size_t term_pos = data.find('\n', offset); + if (term_pos != data.length() - 1 && term_pos != std::string::npos) + log_cmd_error("read failed: more than one response\n"); + } + return data; + } + + ~HandleRpcServer() { + CloseHandle(hsend); + if (hrecv != hsend) + CloseHandle(hrecv); + } +}; + +#else + +struct FdRpcServer : RpcServer { + int fdsend, fdrecv; + pid_t pid; + + FdRpcServer(const std::string &name, int fdsend, int fdrecv, pid_t pid = -1) + : RpcServer(name), fdsend(fdsend), fdrecv(fdrecv), pid(pid) { } + + void check_pid() { + if (pid == -1) return; + // If we're communicating with a process, check that it's still running, or we may get killed with SIGPIPE. + pid_t wait_result = ::waitpid(pid, NULL, WNOHANG); + if (wait_result == -1) + log_cmd_error("waitpid failed: %s\n", strerror(errno)); + if (wait_result == pid) + log_cmd_error("RPC frontend terminated unexpectedly\n"); + } + + void write(const std::string &data) YS_OVERRIDE { + log_assert(data.length() >= 1 && data.find('\n') == data.length() - 1); + ssize_t offset = 0; + do { + check_pid(); + ssize_t result = ::write(fdsend, &data[offset], data.length() - offset); + if (result == -1) + log_cmd_error("write failed: %s\n", strerror(errno)); + offset += result; + } while(offset < (ssize_t)data.length()); + } + + std::string read() YS_OVERRIDE { + std::string data; + ssize_t offset = 0; + while (data.length() == 0 || data[data.length() - 1] != '\n') { + data.resize(data.length() + 1024); + check_pid(); + ssize_t result = ::read(fdrecv, &data[offset], data.length() - offset); + if (result == -1) + log_cmd_error("read failed: %s\n", strerror(errno)); + offset += result; + data.resize(offset); + size_t term_pos = data.find('\n', offset); + if (term_pos != data.length() - 1 && term_pos != std::string::npos) + log_cmd_error("read failed: more than one response\n"); + } + return data; + } + + ~FdRpcServer() { + close(fdsend); + if (fdrecv != fdsend) + close(fdrecv); + } +}; + +#endif + +// RpcFrontend does not inherit from Frontend since it does not read files. +struct RpcFrontend : public Pass { + RpcFrontend() : Pass("connect_rpc", "connect to RPC frontend") { } + void help() YS_OVERRIDE + { + // |---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---|---v---| + log("\n"); + log(" connect_rpc -exec [args...]\n"); + log(" connect_rpc -path \n"); + log("\n"); + log("Load modules using an out-of-process frontend.\n"); + log("\n"); + log(" -exec [args...]\n"); + log(" run with arguments [args...]. send requests on stdin, read\n"); + log(" responses from stdout.\n"); + log("\n"); + log(" -path \n"); + log(" connect to Unix domain socket at . (Unix)\n"); + log(" connect to bidirectional byte-type named pipe at . (Windows)\n"); + log("\n"); + log("A simple JSON-based, newline-delimited protocol is used for communicating with\n"); + log("the frontend. Yosys requests data from the frontend by sending exactly 1 line\n"); + log("of JSON. Frontend responds with data or error message by replying with exactly\n"); + log("1 line of JSON as well.\n"); + log("\n"); + log(" -> {\"method\": \"modules\"}\n"); + log(" <- {\"modules\": [\"\", ...]}\n"); + log(" <- {\"error\": \"\"}\n"); + log(" request for the list of modules that can be derived by this frontend.\n"); + log(" the 'hierarchy' command will call back into this frontend if a cell\n"); + log(" with type is instantiated in the design.\n"); + log("\n"); + log(" -> {\"method\": \"derive\", \"module\": \", \"parameters\": {\n"); + log(" \"\": {\"type\": \"[unsigned|signed|string|real]\",\n"); + log(" \"value\": \"\"}, ...}}\n"); + log(" <- {\"frontend\": \"[ilang|verilog|...]\",\"source\": \"\"}}\n"); + log(" <- {\"error\": \"\"}\n"); + log(" request for the module to be derived for a specific set of\n"); + log(" parameters. starts with \\ for named parameters, and with $\n"); + log(" for unnamed parameters, which are numbered starting at 1.\n"); + log(" for integer parameters is always specified as a binary string of unlimited\n"); + log(" precision. the returned by the frontend is hygienically parsed\n"); + log(" by a built-in Yosys , allowing the RPC frontend to return any\n"); + log(" convenient representation of the module. the derived module is cached,\n"); + log(" so the response should be the same whenever the same set of parameters\n"); + log(" is provided.\n"); + } + void execute(std::vector args, RTLIL::Design *design) YS_OVERRIDE + { + log_header(design, "Connecting to RPC frontend.\n"); + + std::vector command; + std::string path; + size_t argidx; + for (argidx = 1; argidx < args.size(); argidx++) { + std::string arg = args[argidx]; + if (arg == "-exec" && argidx+1 < args.size()) { + command.insert(command.begin(), args.begin() + argidx + 1, args.end()); + continue; + } + if (arg == "-path" && argidx+1 < args.size()) { + path = args[argidx+1]; + continue; + } + break; + } + extra_args(args, argidx, design); + + if ((!command.empty()) + (!path.empty()) != 1) + log_cmd_error("Exactly one of -exec, -unix must be specified.\n"); + + std::shared_ptr server; + if (!command.empty()) { + std::string command_line; + bool first = true; + for (auto &arg : command) { + if (!first) command_line += ' '; + command_line += arg; + first = false; + } + +#ifdef _WIN32 + std::wstring command_w = str2wstr(command[0]); + std::wstring command_path_w; + std::wstring command_line_w = str2wstr(command_line); + DWORD command_path_len_w; + SECURITY_ATTRIBUTES pipe_attr = {}; + HANDLE send_r = NULL, send_w = NULL, recv_r = NULL, recv_w = NULL; + STARTUPINFOW startup_info = {}; + PROCESS_INFORMATION proc_info = {}; + + command_path_len_w = SearchPathW(/*lpPath=*/NULL, /*lpFileName=*/command_w.c_str(), /*lpExtension=*/L".exe", /*nBufferLength=*/0, /*lpBuffer=*/NULL, /*lpFilePart=*/NULL); + if (command_path_len_w == 0) { + log_error("SearchPathW failed: %s\n", get_last_error_str().c_str()); + goto cleanup_exec; + } + command_path_w.resize(command_path_len_w - 1); + command_path_len_w = SearchPathW(/*lpPath=*/NULL, /*lpFileName=*/command_w.c_str(), /*lpExtension=*/L".exe", /*nBufferLength=*/command_path_len_w, /*lpBuffer=*/&command_path_w[0], /*lpFilePart=*/NULL); + log_assert(command_path_len_w == command_path_w.length()); + + pipe_attr.nLength = sizeof(pipe_attr); + pipe_attr.bInheritHandle = TRUE; + pipe_attr.lpSecurityDescriptor = NULL; + if (!CreatePipe(&send_r, &send_w, &pipe_attr, /*nSize=*/0)) { + log_error("CreatePipe failed: %s\n", get_last_error_str().c_str()); + goto cleanup_exec; + } + if (!SetHandleInformation(send_w, HANDLE_FLAG_INHERIT, 0)) { + log_error("SetHandleInformation failed: %s\n", get_last_error_str().c_str()); + goto cleanup_exec; + } + if (!CreatePipe(&recv_r, &recv_w, &pipe_attr, /*nSize=*/0)) { + log_error("CreatePipe failed: %s\n", get_last_error_str().c_str()); + goto cleanup_exec; + } + if (!SetHandleInformation(recv_r, HANDLE_FLAG_INHERIT, 0)) { + log_error("SetHandleInformation failed: %s\n", get_last_error_str().c_str()); + goto cleanup_exec; + } + + startup_info.cb = sizeof(startup_info); + startup_info.hStdInput = send_r; + startup_info.hStdOutput = recv_w; + startup_info.hStdError = GetStdHandle(STD_ERROR_HANDLE); + startup_info.dwFlags |= STARTF_USESTDHANDLES; + if (!CreateProcessW(/*lpApplicationName=*/command_path_w.c_str(), /*lpCommandLine=*/&command_line_w[0], /*lpProcessAttributes=*/NULL, /*lpThreadAttributes=*/NULL, /*bInheritHandles=*/TRUE, /*dwCreationFlags=*/0, /*lpEnvironment=*/NULL, /*lpCurrentDirectory=*/NULL, &startup_info, &proc_info)) { + log_error("CreateProcessW failed: %s\n", get_last_error_str().c_str()); + goto cleanup_exec; + } + CloseHandle(proc_info.hProcess); + CloseHandle(proc_info.hThread); + + server = std::make_shared(path, send_w, recv_r); + send_w = NULL; + recv_r = NULL; + +cleanup_exec: + if (send_r != NULL) CloseHandle(send_r); + if (send_w != NULL) CloseHandle(send_w); + if (recv_r != NULL) CloseHandle(recv_r); + if (recv_w != NULL) CloseHandle(recv_w); +#else + std::vector argv; + int send[2] = {-1,-1}, recv[2] = {-1,-1}; + posix_spawn_file_actions_t file_actions, *file_actions_p = NULL; + pid_t pid; + + for (auto &arg : command) + argv.push_back(&arg[0]); + argv.push_back(nullptr); + + if (pipe(send) != 0) { + log_error("pipe failed: %s\n", strerror(errno)); + goto cleanup_exec; + } + if (pipe(recv) != 0) { + log_error("pipe failed: %s\n", strerror(errno)); + goto cleanup_exec; + } + + if (posix_spawn_file_actions_init(&file_actions) != 0) { + log_error("posix_spawn_file_actions_init failed: %s\n", strerror(errno)); + goto cleanup_exec; + } + file_actions_p = &file_actions; + if (posix_spawn_file_actions_adddup2(file_actions_p, send[0], STDIN_FILENO) != 0) { + log_error("posix_spawn_file_actions_adddup2 failed: %s\n", strerror(errno)); + goto cleanup_exec; + } + if (posix_spawn_file_actions_addclose(file_actions_p, send[1]) != 0) { + log_error("posix_spawn_file_actions_addclose failed: %s\n", strerror(errno)); + goto cleanup_exec; + } + if (posix_spawn_file_actions_adddup2(file_actions_p, recv[1], STDOUT_FILENO) != 0) { + log_error("posix_spawn_file_actions_adddup2 failed: %s\n", strerror(errno)); + goto cleanup_exec; + } + if (posix_spawn_file_actions_addclose(file_actions_p, recv[0]) != 0) { + log_error("posix_spawn_file_actions_addclose failed: %s\n", strerror(errno)); + goto cleanup_exec; + } + + if (posix_spawnp(&pid, argv[0], file_actions_p, /*attrp=*/NULL, argv.data(), environ) != 0) { + log_error("posix_spawnp failed: %s\n", strerror(errno)); + goto cleanup_exec; + } + + server = std::make_shared(command_line, send[1], recv[0], pid); + send[1] = -1; + recv[0] = -1; + +cleanup_exec: + if (send[0] != -1) close(send[0]); + if (send[1] != -1) close(send[1]); + if (recv[0] != -1) close(recv[0]); + if (recv[1] != -1) close(recv[1]); + if (file_actions_p != NULL) + posix_spawn_file_actions_destroy(file_actions_p); +#endif + } else if (!path.empty()) { +#ifdef _WIN32 + std::wstring path_w = str2wstr(path); + HANDLE h; + + h = CreateFileW(path_w.c_str(), GENERIC_READ|GENERIC_WRITE, /*dwShareMode=*/0, /*lpSecurityAttributes=*/NULL, /*dwCreationDisposition=*/OPEN_EXISTING, /*dwFlagsAndAttributes=*/0, /*hTemplateFile=*/NULL); + if (h == INVALID_HANDLE_VALUE) { + log_error("CreateFileW failed: %s\n", get_last_error_str().c_str()); + goto cleanup_path; + } + + server = std::make_shared(path, h, h); + +cleanup_path: + ; +#else + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, path.c_str(), sizeof(addr.sun_path) - 1); + + int fd = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd == -1) { + log_error("socket failed: %s\n", strerror(errno)); + goto cleanup_path; + } + + if (connect(fd, (struct sockaddr *)&addr, sizeof(addr)) != 0) { + log_error("connect failed: %s\n", strerror(errno)); + goto cleanup_path; + } + + server = std::make_shared(path, fd, fd); + fd = -1; + +cleanup_path: + if (fd != -1) close(fd); +#endif + } + + if (!server) + log_cmd_error("Failed to connect to RPC frontend.\n"); + + for (auto &module_name : server->get_module_names()) { + log("Linking module `%s'.\n", module_name.c_str()); + RpcModule *module = new RpcModule; + module->name = "$abstract\\" + module_name; + module->server = server; + design->add(module); + } + } +} RpcFrontend; + +YOSYS_NAMESPACE_END diff --git a/tests/rpc/.gitignore b/tests/rpc/.gitignore new file mode 100644 index 000000000..397b4a762 --- /dev/null +++ b/tests/rpc/.gitignore @@ -0,0 +1 @@ +*.log diff --git a/tests/rpc/design.v b/tests/rpc/design.v new file mode 100644 index 000000000..80f1dac1a --- /dev/null +++ b/tests/rpc/design.v @@ -0,0 +1,8 @@ +module top(input [3:0] i, output [3:0] o); + python_inv #( + .width(4) + ) inv ( + .i(i), + .o(o), + ); +endmodule diff --git a/tests/rpc/exec.ys b/tests/rpc/exec.ys new file mode 100644 index 000000000..b46009fb9 --- /dev/null +++ b/tests/rpc/exec.ys @@ -0,0 +1,5 @@ +connect_rpc -exec python3 frontend.py stdio +read_verilog design.v +hierarchy -top top +flatten +select -assert-count 1 t:$neg diff --git a/tests/rpc/frontend.py b/tests/rpc/frontend.py new file mode 100644 index 000000000..eff41738a --- /dev/null +++ b/tests/rpc/frontend.py @@ -0,0 +1,126 @@ +def modules(): + return ["python_inv"] + +def derive(module, parameters): + assert module == r"python_inv" + if parameters.keys() != {r"\width"}: + raise ValueError("Invalid parameters") + return "ilang", r""" +module \impl + wire width {width:d} input 1 \i + wire width {width:d} output 2 \o + cell $neg $0 + parameter \A_SIGNED 1'0 + parameter \A_WIDTH 32'{width:b} + parameter \Y_WIDTH 32'{width:b} + connect \A \i + connect \Y \o + end +end +module \python_inv + wire width {width:d} input 1 \i + wire width {width:d} output 2 \o + cell \impl $0 + connect \i \i + connect \o \o + end +end +""".format(width=parameters[r"\width"]) + +# ---------------------------------------------------------------------------- + +import json +import argparse +import sys, socket, os +try: + import msvcrt, win32pipe, win32file +except ImportError: + msvcrt = win32pipe = win32file = None + +def map_parameter(parameter): + if parameter["type"] == "unsigned": + return int(parameter["value"], 2) + if parameter["type"] == "signed": + width = len(parameter["value"]) + value = int(parameter["value"], 2) + if value & (1 << (width - 1)): + value = -((1 << width) - value) + return value + if parameter["type"] == "string": + return parameter["value"] + if parameter["type"] == "real": + return float(parameter["value"]) + +def call(input_json): + input = json.loads(input_json) + if input["method"] == "modules": + return json.dumps({"modules": modules()}) + if input["method"] == "derive": + try: + frontend, source = derive(input["module"], + {name: map_parameter(value) for name, value in input["parameters"].items()}) + return json.dumps({"frontend": frontend, "source": source}) + except ValueError as e: + return json.dumps({"error": str(e)}) + +def main(): + parser = argparse.ArgumentParser() + modes = parser.add_subparsers(dest="mode") + mode_stdio = modes.add_parser("stdio") + if os.name == "posix": + mode_path = modes.add_parser("unix-socket") + if os.name == "nt": + mode_path = modes.add_parser("named-pipe") + mode_path.add_argument("path") + args = parser.parse_args() + + if args.mode == "stdio": + while True: + input = sys.stdin.readline() + if not input: break + sys.stdout.write(call(input) + "\n") + sys.stdout.flush() + + if args.mode == "unix-socket": + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.bind(args.path) + try: + sock.listen(1) + conn, addr = sock.accept() + file = conn.makefile("rw") + while True: + input = file.readline() + if not input: break + file.write(call(input) + "\n") + file.flush() + finally: + sock.close() + os.unlink(args.path) + + if args.mode == "named-pipe": + pipe = win32pipe.CreateNamedPipe(args.path, win32pipe.PIPE_ACCESS_DUPLEX, + win32pipe.PIPE_TYPE_BYTE|win32pipe.PIPE_READMODE_BYTE|win32pipe.PIPE_WAIT, + 1, 4096, 4096, 0, None) + win32pipe.ConnectNamedPipe(pipe, None) + try: + while True: + input = b"" + while not input.endswith(b"\n"): + result, data = win32file.ReadFile(pipe, 4096) + assert result == 0 + input += data + assert not b"\n" in input or input.endswith(b"\n") + output = (call(input.decode("utf-8")) + "\n").encode("utf-8") + length = len(output) + while length > 0: + result, done = win32file.WriteFile(pipe, output) + assert result == 0 + length -= done + except win32file.error as e: + if e.args[0] == 109: # ERROR_BROKEN_PIPE + pass + else: + raise + +if __name__ == "__main__": + main() diff --git a/tests/rpc/run-test.sh b/tests/rpc/run-test.sh new file mode 100755 index 000000000..44ce7e674 --- /dev/null +++ b/tests/rpc/run-test.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e +for x in *.ys; do + echo "Running $x.." + ../../yosys -ql ${x%.ys}.log $x +done diff --git a/tests/rpc/unix.ys b/tests/rpc/unix.ys new file mode 100644 index 000000000..cc7ec14ab --- /dev/null +++ b/tests/rpc/unix.ys @@ -0,0 +1,6 @@ +!python3 frontend.py unix-socket frontend.sock & sleep 0.1 +connect_rpc -path frontend.sock +read_verilog design.v +hierarchy -top top +flatten +select -assert-count 1 t:$neg -- cgit v1.2.3