MAGICL extensions refactor

.gitignore

@@ -1,2 +1,2 @@
-transcendental/libexpokit.dylib
-transcendental/libexpokit.so
+**/*.dylib
+**/*.so

README.md

@@ -3,12 +3,15 @@
 
 _Matrix Algebra proGrams In Common Lisp_ by [Rigetti Computing](http://www.rigetti.com). (née FLAIL: _Finally, Linear Algebra In Lisp!_)
 
-(**Note**: The high-level interface is experimental and subject to rapid change.)
+(**Note**: The high-level interface is experimental and subject to change.)
 
 ## Requirements
 
  * SBCL (> 1.3.19) or CCL (>= 1.11) on AMD64
  * quicklisp
+
+If you'll want to use BLAS/LAPACK for extended functionality, you'll need:
+
  * libffi
  * BLAS and LAPACK
 
@@ -29,18 +32,34 @@ running `sbcl` and evaluating `ql:*local-project-directories*`. Once
 installed, confirm that MAGICL is working properly by running the
 tests, as described in the next section.
 
+## Lisp-Only vs Accelerated MAGICL
+
+MAGICL by default only uses pure ANSI Common Lisp code. If you with to
+accelerate it or extend the functionality, you may load *MAGICL
+extensions*. The available ones are:
+
+- `MAGICL/FANCY`: for every extension we've got under the kitchen sink
+- `MAGICL/EXT-BLAS`: for BLAS functions
+- `MAGICL/EXT-LAPACK`: for LAPACK functions
+- `MAGICL/EXT-EXPOKIT`: for expokit (matrix `exp()`) functions
+
+If you use extensions, you'll need the requisite C/Fortran
+libraries. Expokit will automatically build for you, as it's included
+in the distribution of MAGICL.
+
 ## Testing MAGICL
 
 You can run the MAGICL tests from your Lisp REPL with:
 
 ```
-(ql:quickload :magicl-tests)
 (asdf:test-system :magicl)
 ```
 
+You currently need all of the extensions working for the tests to run.
+
 ## High-level Interface
 
-See [high-level doc](doc/high-level.md).
+See [high-level doc](doc/high-level.md) for an extensive discussion and comparison of MAGICL functions with those of MATLAB and NumPy.
 
 ## Showing Available Functions
 

examples.lisp

@@ -1,9 +1,5 @@
 (defpackage #:magicl-examples
-  (:use #:common-lisp #:magicl-transcendental)
-  #+package-local-nicknames
-  (:local-nicknames (:blas :magicl.blas-cffi)
-                    (:lapack :magicl.lapack-cffi)
-                    (:expokit :magicl.expokit-cffi))
+  (:use #:common-lisp)
   (:export #:dot-example
            #:eigenvalue-example
            #:qr-example
@@ -137,8 +133,8 @@
     (format t "X*X^-1~%~a~%" id)))
 
 (defun expm-example ()
-  (let* ((x  (magicl:from-list (list 0 0 0 0 0 0 1.5 0 0 -1.5 0 0.5 0 0 -0.5 0) '(4 4) :type 'double-float))
-         (expx (magicl-transcendental:expm x))
+  (let* ((x  (magicl:from-list (list 0 0 0 0 0 0 1.5 0 0 -1.5 0 0.5 0 0 -0.5 0) '(4 4) :type '(complex double-float)))
+         (expx (magicl:expm x))
          (d (magicl:det expx)))
     (format t "X~%~a~%" x)
     (format t "e^X~%~a~%" expx)

magicl-examples.asd

@@ -3,7 +3,7 @@
   :description "MAGICL examples"
   :maintainer "Rigetti Computing"
   :author "Rigetti Computing"
-  :depends-on (#:magicl #:magicl-transcendental)
+  :depends-on (#:magicl/fancy)
   :serial t
   :components
   ((:file "examples")))

magicl-tests.asd

@@ -8,8 +8,7 @@
   :author "Rigetti Computing"
   :depends-on (#:alexandria
                #:uiop
-               #:magicl
-               #:magicl-transcendental
+               #:magicl/fancy
                #:magicl-examples
                #:fiasco)
   :perform (asdf:test-op (o s)

magicl.asd

@@ -9,31 +9,19 @@
   :author "Rigetti Computing"
   :version (:read-file-form "VERSION.txt")
   :depends-on (#:alexandria
-               #:cffi
-               #:cffi-libffi
                #:abstract-classes
-               #:policy-cond)
+               #:policy-cond
+
+               #:magicl/ext             ; Allow extensions
+               )
   :in-order-to ((asdf:test-op (asdf:test-op #:magicl-tests)))
   :around-compile (lambda (compile)
                     (let (#+sbcl (sb-ext:*derive-function-types* t))
                       (funcall compile)))
-  :serial t
   :pathname "src/"
+  :serial t
   :components
   ((:file "packages")
-   (:file "load-libs")
-   (:file "with-array-pointers")
-   (:file "cffi-types")
-   (:module "bindings"
-    :components ((:file "lapack00-cffi")
-                 (:file "lapack01-cffi")
-                 (:file "lapack02-cffi")
-                 (:file "lapack03-cffi")
-                 (:file "lapack04-cffi")
-                 (:file "lapack05-cffi")
-                 (:file "lapack06-cffi")
-                 (:file "lapack07-cffi")
-                 (:file "blas-cffi")))
    (:module "high-level"
     :serial t
     :components ((:file "util")
@@ -43,15 +31,141 @@
                  (:file "tensor")
                  (:file "matrix")
                  (:file "vector")
-                 (:file "lapack-generics")
                  (:file "types/single-float")
                  (:file "types/double-float")
                  (:file "types/complex-single-float")
                  (:file "types/complex-double-float")
                  (:file "types/int32")
-                 (:file "lapack-templates")
-                 (:file "lapack-bindings")
                  (:file "constructors")
                  (:file "specialize-constructor")
                  (:file "polynomial-solver")))
    (:file "magicl")))
+
+;;; Extension common code
+
+(asdf:defsystem #:magicl/ext
+  :description "Common code for extending MAGICL with foreign libraries."
+  :depends-on (#:cffi
+               #:cffi-libffi)
+  :serial t
+  :pathname "src/extensions/common"
+  :components
+  ((:file "package")
+   (:file "library-tracking")
+   (:file "with-array-pointers")
+   (:file "cffi-types")))
+
+;;; BLAS
+
+(asdf:defsystem #:magicl/ext-blas
+  :description "Native BLAS routines in MAGICL."
+  :depends-on (#:magicl/ext
+               #:cffi)
+  :serial t
+  :pathname "src/"
+  :components
+  ((:file "extensions/blas/package")
+   (:file "extensions/blas/load-libs")
+   (:module "bindings"
+    :components ((:file "blas-cffi")))))
+
+
+;;; LAPACK
+
+(asdf:defsystem #:magicl/ext-lapack
+  :description "Native LAPACK routines in MAGICL."
+  :depends-on (#:magicl
+               #:magicl/ext
+               #:magicl/ext-blas
+               #:cffi
+               #:policy-cond)
+  :serial t
+  :pathname "src/"
+  :components
+  ((:file "extensions/lapack/package")
+   (:file "extensions/lapack/load-libs")
+   (:module "bindings"
+    :components ((:file "lapack00-cffi")
+                 (:file "lapack01-cffi")
+                 (:file "lapack02-cffi")
+                 (:file "lapack03-cffi")
+                 (:file "lapack04-cffi")
+                 (:file "lapack05-cffi")
+                 (:file "lapack06-cffi")
+                 (:file "lapack07-cffi")))
+   (:module "extensions/lapack"
+    :components ((:file "lapack-generics")
+                 (:file "lapack-templates")
+                 (:file "lapack-bindings")
+                 (:file "lapack-csd")))))
+
+
+;;; EXPOKIT
+
+;;; Adapted from commonqt's qt.asd
+(defclass f->so (asdf:source-file)
+  ()
+  (:default-initargs
+   :type "f"))
+
+(defmethod output-files ((operation compile-op) (component f->so))
+  (values (list (make-pathname :name "libexpokit"
+                               :type #-darwin "so" #+darwin "dylib"
+                               :defaults (component-pathname component)))
+          t))
+
+(defmethod perform ((operation load-op) (component f->so))
+  t)
+
+(defmethod perform ((operation compile-op) (component f->so))
+  (flet ((nn (x) (uiop:native-namestring x)))
+    (let* ((fortran-file (component-pathname component))
+           (object-file (make-pathname :type "o" :defaults fortran-file))
+           (shared-object (make-pathname :type #+darwin "dylib" #-darwin "so"
+                                         :name "libexpokit"
+                                         :defaults fortran-file)))
+      (uiop:run-program
+       (list "gfortran" "-fPIC" "-std=legacy"
+             "-c"
+             (nn fortran-file)
+             "-o"
+             (nn object-file)))
+      (uiop:run-program
+       (list "gfortran" #+darwin "-dynamiclib" #-darwin "-shared"
+             "-o" (nn shared-object)
+             (nn object-file)
+             #+darwin "-lblas"
+             #+darwin "-llapack"))
+      (delete-file object-file))))
+
+
+(asdf:defsystem #:magicl/ext-expokit
+  :description "Expokit for MAGICL."
+  :depends-on (#:alexandria
+               #:cffi
+               #:cffi-libffi
+               #:magicl
+               #:magicl/ext
+               #:magicl/ext-blas
+               #:magicl/ext-lapack)
+  :serial t
+  :components
+  ((:module "expokit"
+    :components ((f->so "expokit")))
+   (:module "src/extensions/expokit"
+    :components ((:file "package")
+                 (:file "load-libs")))
+   (:file "src/bindings/expokit-cffi")
+   (:file "src/extensions/expokit/expm")))
+
+
+;;; Fancy - Load everything
+
+(asdf:defsystem #:magicl/fancy
+  :description "MAGICL with all extensions."
+  :depends-on (#:magicl
+               #:magicl/ext-blas
+               #:magicl/ext-lapack
+               #:magicl/ext-expokit)
+  :serial t
+  :components ())

src/extensions/blas/load-libs.lisp

@@ -0,0 +1,19 @@
+(in-package #:magicl.foreign-libraries)
+
+(cffi:define-foreign-library libblas
+  #+:magicl.use-accelerate
+  (:darwin "libBLAS.dylib" :search-path #P"/System/Library/Frameworks/Accelerate.framework/Frameworks/vecLib.framework/Versions/A/")
+  #-:magicl.use-accelerate
+  (:darwin (:or "/usr/local/opt/lapack/lib/libblas.dylib" "libblas.dylib" ))
+  #+:magicl.use-mkl
+  (:unix  "libmkl_rt.so")
+  #-:magicl.use-mkl
+  (:unix  (:or "libblas.so"
+               "libblas.so.3"))
+  (t (:default "libblas")))
+
+(defvar *blas-loaded* nil)
+
+(unless *blas-loaded*
+  (cffi:load-foreign-library 'libblas)
+  (setf *blas-loaded* t))

src/extensions/blas/package.lisp

@@ -0,0 +1,6 @@
+
+(defpackage #:magicl.blas-cffi
+  (:use)
+  #-package-local-nicknames
+  (:nicknames #:blas))
+