From 66447ef5bd2cb1c4c8ea113807698cd27c4493cc Mon Sep 17 00:00:00 2001 From: Owen Lynch Date: Mon, 22 Jan 2024 20:18:52 -0800 Subject: [PATCH 1/5] WIP more docstrings for intertypes --- src/intertypes/InterTypes.jl | 76 +++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/src/intertypes/InterTypes.jl b/src/intertypes/InterTypes.jl index 243d4f0..273c0e8 100644 --- a/src/intertypes/InterTypes.jl +++ b/src/intertypes/InterTypes.jl @@ -7,6 +7,15 @@ using OrderedCollections using ..Schemas import ..Schemas: toexpr +# InterType Definitions +####################### + +""" +A field of a struct. Used in [`Variant`](@ref) and [`Record`](@ref). + +The `T` parameter will always be [`InterType`](@ref), but this is mutually-recursive +with `InterType` so we have to be generic here. +""" struct Field{T} name::Symbol type::T @@ -14,6 +23,12 @@ end Base.nameof(field::Field) = field.name +""" +One of the summands of a sum type. + +Like [`Field`](@ref), the `T` parameter will always be [`InterType`](@ref), but +this is mutually-recursive with `InterType` so we have to be generic here. +""" struct Variant{T} tag::Symbol fields::Vector{Field{T}} @@ -21,8 +36,12 @@ end Base.nameof(variant::Variant) = variant.tag +""" +A non-empty linked list of symbols representing something like `foo.bar.baz`. +""" @data RefPath begin RefHere(name::Symbol) + """mod.name""" RefThere(mod::RefPath, name::Symbol) end @@ -44,6 +63,21 @@ function toexpr(p::RefPath) end end +""" +An intertype expression representing a type. + +TODO: Generic types +TODO: Remove anonymous sums, anonymous products +TODO: Separate out primitives, so that this is something like + +```julia +@data InterType begin + PrimType(prim::InterTypePrimitive) + TypeRef(path::RefPath) + TypeApp(type::InterType, args::Vector{InterType}) +end +``` +""" @data InterType begin Unit I32 @@ -61,11 +95,27 @@ end OptionalType(elemtype::InterType) Record(fields::Vector{Field{InterType}}) Sum(variants::Vector{Variant{InterType}}) - ACSetInterType(schema::TypedSchema{InterType}) + ACSetInterType(schema::TypedSchema{Symbol, InterType}) Annot(desc::String, type::InterType) TypeRef(to::RefPath) end +""" +A specification for the type of an acset. + +## Fields +- `genericname::Union{Symbol, Nothing}`: The name for the generic version of the acset, with type parameters. + + Note that the name assigned to this in the declaration is the name *with* + type parameters pre-specified. + + If there are no attribute types, then this is nothing. +- `abstract_type::Union{Symbol, Nothing}`: The parent abstract type for the acset. +- `schemaname::Symbol` +- `schema::TypedSchema{Symbol, InterType}` +- `index::Vector{Symbol}` +- `unique_index::Vector{Symbol}` +""" struct ACSetTypeSpec genericname::Union{Symbol, Nothing} abstract_type::Union{Symbol, Nothing} @@ -75,13 +125,37 @@ struct ACSetTypeSpec unique_index::Vector{Symbol} end +""" +An intertype declaration. + +Does not include the name of the declaration. +""" @data InterTypeDecl begin + """An alias for an existing type""" Alias(type::InterType) + """A sum type, also known as a tagged union.""" SumType(variants::Vector{Variant{InterType}}) + """ + A variant of a sum type, i.e. one of the summands. These are implicitly + produced when declaring a sum type, and the data of the variant (i.e. the + fields) are in the parent sum type. + """ VariantOf(parent::Symbol) + """A struct type, also known as a product type or record type.""" Struct(fields::Vector{Field{InterType}}) + """ + A schema for acsets. Does not declare the acset type yet, however, that is + done by [`NamedACSetType`](@ref). + """ SchemaDecl(schema::TypedSchema{Symbol, InterType}) + """ + An abstract acset type which ACSets can subtype. Mostly used for backwards + compatibility with AlgebraicJulia code. + """ AbstractACSetType(parent::Union{Symbol, Nothing}) + """ + An acset type, customized in certain ways. + """ NamedACSetType(typespec::ACSetTypeSpec) end From 93fb9e7f6ca0c20c300c02a7d5c0d897e3ab80eb Mon Sep 17 00:00:00 2001 From: Owen Lynch Date: Tue, 23 Jan 2024 15:03:04 -0800 Subject: [PATCH 2/5] finished commenting InterTypes.jl --- src/intertypes/InterTypes.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/intertypes/InterTypes.jl b/src/intertypes/InterTypes.jl index 273c0e8..5ea4ac2 100644 --- a/src/intertypes/InterTypes.jl +++ b/src/intertypes/InterTypes.jl @@ -171,6 +171,10 @@ end function hashdecls end +""" +A collection of intertype declarations packaged together. May refer to other +InterTypeModules via the `imports` field. +""" struct InterTypeModule name::Symbol imports::OrderedDict{Symbol, InterTypeModule} From dce0208ca5be7a16dbea149fa4ae0999a52e7ffc Mon Sep 17 00:00:00 2001 From: Owen Lynch Date: Tue, 23 Jan 2024 15:15:16 -0800 Subject: [PATCH 3/5] moved docstrings out of @data macro --- src/intertypes/InterTypes.jl | 98 ++++++++++++++++++++---------------- src/intertypes/json.jl | 1 - 2 files changed, 55 insertions(+), 44 deletions(-) diff --git a/src/intertypes/InterTypes.jl b/src/intertypes/InterTypes.jl index 5ea4ac2..7c62f13 100644 --- a/src/intertypes/InterTypes.jl +++ b/src/intertypes/InterTypes.jl @@ -36,15 +36,18 @@ end Base.nameof(variant::Variant) = variant.tag -""" -A non-empty linked list of symbols representing something like `foo.bar.baz`. -""" @data RefPath begin RefHere(name::Symbol) - """mod.name""" RefThere(mod::RefPath, name::Symbol) end +@doc """ +A non-empty linked list of symbols representing something like `foo.bar.baz`. +""" RefPath + +@doc """E.g. mod.name""" RefThere + + function RefPath(s::Symbol) RefHere(s) end @@ -63,21 +66,6 @@ function toexpr(p::RefPath) end end -""" -An intertype expression representing a type. - -TODO: Generic types -TODO: Remove anonymous sums, anonymous products -TODO: Separate out primitives, so that this is something like - -```julia -@data InterType begin - PrimType(prim::InterTypePrimitive) - TypeRef(path::RefPath) - TypeApp(type::InterType, args::Vector{InterType}) -end -``` -""" @data InterType begin Unit I32 @@ -100,6 +88,22 @@ end TypeRef(to::RefPath) end +@doc """ +An intertype expression representing a type. + +TODO: Generic types +TODO: Remove anonymous sums, anonymous products +TODO: Separate out primitives, so that this is something like + +```julia +@data InterType begin + PrimType(prim::InterTypePrimitive) + TypeRef(path::RefPath) + TypeApp(type::InterType, args::Vector{InterType}) +end +``` +""" InterType + """ A specification for the type of an acset. @@ -125,40 +129,48 @@ struct ACSetTypeSpec unique_index::Vector{Symbol} end -""" -An intertype declaration. - -Does not include the name of the declaration. -""" @data InterTypeDecl begin - """An alias for an existing type""" Alias(type::InterType) - """A sum type, also known as a tagged union.""" SumType(variants::Vector{Variant{InterType}}) - """ - A variant of a sum type, i.e. one of the summands. These are implicitly - produced when declaring a sum type, and the data of the variant (i.e. the - fields) are in the parent sum type. - """ VariantOf(parent::Symbol) - """A struct type, also known as a product type or record type.""" Struct(fields::Vector{Field{InterType}}) - """ - A schema for acsets. Does not declare the acset type yet, however, that is - done by [`NamedACSetType`](@ref). - """ SchemaDecl(schema::TypedSchema{Symbol, InterType}) - """ - An abstract acset type which ACSets can subtype. Mostly used for backwards - compatibility with AlgebraicJulia code. - """ AbstractACSetType(parent::Union{Symbol, Nothing}) - """ - An acset type, customized in certain ways. - """ NamedACSetType(typespec::ACSetTypeSpec) end +@doc """ +An intertype declaration. + +Does not include the name of the declaration. +""" InterTypeDecl + +@doc """An alias for an existing type""" Alias + +@doc """A sum type, also known as a tagged union.""" SumType + +@doc """ +A variant of a sum type, i.e. one of the summands. These are implicitly +produced when declaring a sum type, and the data of the variant (i.e. the +fields) are in the parent sum type. +""" VariantOf + +@doc """A struct type, also known as a product type or record type.""" Struct + +@doc """ +A schema for acsets. Does not declare the acset type yet, however, that is +done by [`NamedACSetType`](@ref). +""" SchemaDecl + +@doc """ +An abstract acset type which ACSets can subtype. Mostly used for backwards +compatibility with AlgebraicJulia code. +""" AbstractACSetType + +@doc """ +An acset type, with customizations like choice of indices, etc. +""" NamedACSetType + exports(::InterTypeDecl) = Symbol[] function exports(acset_type::NamedACSetType) diff --git a/src/intertypes/json.jl b/src/intertypes/json.jl index 22632f4..afbd014 100644 --- a/src/intertypes/json.jl +++ b/src/intertypes/json.jl @@ -115,7 +115,6 @@ intertype(::Type{Optional{T}}) where {T} = Optional{intertype(T)} read(::JSONFormat, ::Type{Optional{T}}, ::Nothing) where {T} = nothing read(format::JSONFormat, ::Type{Optional{T}}, s) where {T} = read(format, T, s) -write(io::IO, format::JSONFormat, d::Nothing) = print(io, "null") intertype(::Type{OrderedDict{K,V}}) where {K,V} = Map(intertype(K), intertype(V)) function read(format::JSONFormat, ::Type{OrderedDict{K, V}}, s::JSON3.Array) where {K, V} From f27a52c92f6751a66d0d4812e927b4e15e7e87a2 Mon Sep 17 00:00:00 2001 From: Owen Lynch Date: Tue, 23 Jan 2024 16:00:15 -0800 Subject: [PATCH 4/5] finished commenting json.jl --- src/intertypes/InterTypes.jl | 2 ++ src/intertypes/json.jl | 70 ++++++++++++++++++++++++++++++++++-- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/src/intertypes/InterTypes.jl b/src/intertypes/InterTypes.jl index 7c62f13..6b1da98 100644 --- a/src/intertypes/InterTypes.jl +++ b/src/intertypes/InterTypes.jl @@ -216,6 +216,8 @@ abstract type ExportTarget end abstract type LanguageTarget <: ExportTarget end abstract type SerializationTarget <: ExportTarget end +abstract type SerializationFormat end + """ generate_module(mod::Module, target::Type{<:ExportTarget}, path="."; target_specific_args...) diff --git a/src/intertypes/json.jl b/src/intertypes/json.jl index afbd014..850a665 100644 --- a/src/intertypes/json.jl +++ b/src/intertypes/json.jl @@ -9,7 +9,10 @@ using CompTime using ..ACSetInterface using ..DenseACSets -struct JSONFormat +""" +A signifier for serialization to and from JSON. +""" +struct JSONFormat <: SerializationFormat end struct InterTypeConversionError <: Exception @@ -17,10 +20,33 @@ struct InterTypeConversionError <: Exception got::Any end -function read(format, type::Type, x) +""" + read(format::SerializationFormat, type::Type, x) + +A generic read method dispatch-controlled by format of the input and expected +type. + +TODO: Currently this *shadows* Base.read, instead of overloading it. Would it +be type piracy to overload it instead? +""" +function read(format::SerializationFormat, type::Type, x) throw(InterTypeConversionError(intertype(type), x)) end +""" + write(io::IO, format::SerializationFormat, x) + +Writes `x` the the provided IO stream, using serialization format `format`. + +See docstring for [`read`](@ref) about shadowing vs. overloading Base.write. +""" +function write end + +""" + joinwith(io::IO, f, xs, separator) + +Prints the elements in `xs` using `x -> f(io, x)`, separated by `separator`. +""" function joinwith(io::IO, f, xs, separator) for x in xs[1:end-1] f(io, x) @@ -29,13 +55,38 @@ function joinwith(io::IO, f, xs, separator) f(io, xs[end]) end +""" + jsonwrite(x) + jsonwrite(io::IO, x) + +A wrapper around [`write`](@ref) which calls it with [`JSONFormat`](@ref). + +TODO: Should we just overload JSON3.write instead? See [1] + +[1]: https://github.com/AlgebraicJulia/ACSets.jl/issues/83 +""" jsonwrite(x) = sprint(jsonwrite, x) jsonwrite(io::IO, x) = write(io, JSONFormat(), x) + +""" + jsonread(s::String, T::Type) + +A wrapper around [`read`](@ref) which calls it with [`JSONFormat`](@ref) + +TODO: See comment for [`jsonwrite`](@ref) +""" function jsonread(s::String, ::Type{T}) where {T} json = JSON3.read(s) read(JSONFormat(), T, json) end +# JSON Serialization for basic types +#################################### + +# We write our own serialization methods instead of using JSON3.write directly +# because we want to serialize 64 bit integers as strings rather than integers +# which get lossily converted to floats. + intertype(::Type{Nothing}) = Unit read(::JSONFormat, ::Type{Nothing}, ::Nothing) = nothing write(io::IO, ::JSONFormat, ::Nothing) = print(io, "null") @@ -262,12 +313,23 @@ function write(io::IO, format::JSONFormat, acs::ACSet) end end +# JSONSchema Export +################### + function fieldproperties(fields::Vector{Field{InterType}}) map(fields) do field field.name => tojsonschema(field.type) end end +""" + tojsonschema(type::InterType) + +Convert an InterType to a JSONSchema representation. + +TODO: We could use multiple dispatch instead of the `@match` here, which might +be cleaner +""" function tojsonschema(type::InterType) @match type begin I32 => Object( @@ -390,9 +452,13 @@ end Specifies a serialization target of JSON Schema when generating a module. + +TODO: This should really be called something like JSONSchemaTarget. """ struct JSONTarget <: SerializationTarget end +# TODO: Should this be ::JSONTarget instead of ::Type{JSONTarget} so +# that we pass in `JSONTarget()` instead of `JSONTarget`? function generate_module( mod::InterTypeModule, ::Type{JSONTarget}, path ;ac=JSON3.AlignmentContext(indent=2) From 01ff30dd9b10f5c1edc178d7509bdd92335bd4ff Mon Sep 17 00:00:00 2001 From: Owen Lynch Date: Tue, 23 Jan 2024 16:53:10 -0800 Subject: [PATCH 5/5] more docstrings --- src/intertypes/julia.jl | 25 ++++++++++++++++++++++++- src/intertypes/python.jl | 11 +++++++++++ src/intertypes/sexp.jl | 6 ++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/intertypes/julia.jl b/src/intertypes/julia.jl index 7284502..9f3912f 100644 --- a/src/intertypes/julia.jl +++ b/src/intertypes/julia.jl @@ -1,6 +1,5 @@ using ...ACSets - function parse_fields(fieldexprs; mod) map(enumerate(fieldexprs)) do (i, fieldexpr) @match fieldexpr begin @@ -165,7 +164,13 @@ function parse_intertype_decl(e; mod::InterTypeModule) end end +""" +We use these macros so that we don't have to, for instance, pattern match on +`Expr(:macrocall, var"@sum", _, args...)`, and can instead pattern match on +`Expr(:sum, args...)`. +""" module InterTypeDeclImplPrivate + macro sum(head, body) esc(Expr(:sum, head, body)) end @@ -183,6 +188,17 @@ macro abstract_acset_type(head) end end +""" + function toexpr(x) end + +Used to convert intertype data types to `Expr`. + +TODO: Should we unify [`tojsonschema`](@ref), [`toexpr`](@ref), and +[`topy`](@ref) into a single function with an extra argument to control +dispatch? +""" +function toexpr end + function toexpr(field::Field) Expr(:(::), field.name, toexpr(field.type)) end @@ -324,6 +340,13 @@ function include_intertypes(into::Module, file::String, imports::AbstractVector) mod end +""" + @intertypes "file.it" module modulename end + +Used to import an intertypes file into Julia. + +TODO: maybe we should just build a .jl file from an .it file directly. +""" macro intertypes(file, modexpr) name, imports = @match modexpr begin Expr(:module, _, name, body) => begin diff --git a/src/intertypes/python.jl b/src/intertypes/python.jl index 6ddfc5b..21001ad 100644 --- a/src/intertypes/python.jl +++ b/src/intertypes/python.jl @@ -1,5 +1,16 @@ export PydanticTarget +""" + topy(intertype::InterType; forward_ref=true) + +Converts an intertype into a python code string. + +TODO: See comment for [`toexpr`](@ref) + +TODO: Should we use something like a stringbuilder instead of manually +concatenating strings? I.e., a tree of strings with O(1) append/splice, +that we write out to a single string at the end? +""" function topy(intertype::InterType; forward_ref=true) @match intertype begin I32 => "int" diff --git a/src/intertypes/sexp.jl b/src/intertypes/sexp.jl index f7ba57f..118bb4f 100644 --- a/src/intertypes/sexp.jl +++ b/src/intertypes/sexp.jl @@ -1,6 +1,12 @@ using SHA using Random +""" +A very simple s-expression data structure. + +We use this to write the intertype schema to a string and then hash it to get a +version identifier. +""" struct SExp args::Vector{Union{Symbol, SExp}} end