Merge pull request #716 from github/this-branch-name-left-intentionally-blank

Ingest Python

Author: robrix

semantic-analysis/semantic-analysis.cabal

@@ -78,6 +78,7 @@ library
     , filepath
     , fused-effects               ^>= 1.1
     , hashable
+    , language-python
     , semantic-source             ^>= 0.2
     , text                        ^>= 1.2.3.1
     , transformers                ^>= 0.5

semantic-analysis/src/Analysis/Syntax/Python.hs

@@ -1,82 +1,122 @@
-{-# LANGUAGE DeriveTraversable #-}
 {-# LANGUAGE FlexibleContexts #-}
 {-# LANGUAGE LambdaCase #-}
 -- | This belongs in @semantic-python@ instead of @semantic-analysis@, but for the sake of expedience…
 module Analysis.Syntax.Python
 ( -- * Syntax
-  Term
-, Python(..)
+  Term(..)
+, subterms
   -- * Abstract interpretation
 , eval0
 , eval
+  -- * Parsing
+, parse
 ) where
 
 import           Analysis.Effect.Domain hiding ((:>>>))
 import qualified Analysis.Effect.Statement as S
 import           Analysis.Name
 import           Analysis.Reference
-import qualified Analysis.Syntax as T
 import           Analysis.VM
 import           Control.Effect.Labelled
 import           Control.Effect.Reader
+import           Control.Monad (foldM)
+import           Data.Foldable (for_)
 import           Data.Function (fix)
-import           Data.List.NonEmpty (NonEmpty)
-import           Data.Text (Text)
-import           Source.Span (Span)
+import           Data.List.NonEmpty (nonEmpty)
+import           Data.Maybe (mapMaybe)
+import qualified Data.Set as Set
+import           Data.Text (pack)
+import qualified Language.Python.Common as Py
+import           Language.Python.Version3.Parser
+import           Source.Span (Pos (..), Span (..), point)
+import           System.FilePath (takeBaseName)
 
 -- Syntax
 
-type Term = T.Term Python Name
+data Term
+  = Module (Py.Module Py.SrcSpan)
+  | Statement (Py.Statement Py.SrcSpan)
+  | Expr (Py.Expr Py.SrcSpan)
+  deriving (Eq, Ord, Show)
 
-data Python t
-  = Noop
-  | Iff t t t
-  | Bool Bool
-  | String Text
-  | Throw t
-  | Let Name t t
-  | t :>> t
-  | Import (NonEmpty Text)
-  | Function Name [Name] t
-  | Call t [t]
-  | Locate Span t
-  deriving (Eq, Foldable, Functor, Ord, Show, Traversable)
-
-infixl 1 :>>
+-- | Non-generic production of the recursive set of subterms.
+--
+-- This should be exactly the set of nodes which 'eval' can visit, i.e. it excludes comments, etc.
+subterms :: Term -> Set.Set Term
+subterms t = Set.insert t $ case t of
+  Module (Py.Module ss)                 -> suite ss
+  Statement (Py.Conditional cts e _)    -> foldMap (\ (c, t) -> subterms (Expr c) <> suite t) cts <> suite e
+  Statement (Py.Raise (Py.RaiseV3 e) _) -> maybe Set.empty (subterms . Expr . fst) e
+  -- FIXME: Py.RaiseV2
+  -- FIXME: whatever the second field is
+  Statement (Py.StmtExpr e _)           -> subterms (Expr  e)
+  Statement (Py.Fun _ _ _ ss _)         -> suite ss
+  -- FIXME: include 'subterms' of any default values
+  Expr (Py.Call f as _)                 -> subterms (Expr f) <> foldMap (\case { Py.ArgExpr e _ -> subterms (Expr e) ; _ -> Set.empty }) as
+  -- FIXME: support keyword args &c.
+  _                                     -> Set.empty -- TBD, and terminals
+  where
+  suite = foldMap (subterms . Statement)
 
 
 -- Abstract interpretation
 
-eval0 :: (Has (Env addr) sig m, HasLabelled Store (Store addr val) sig m, Has (Dom val) sig m, Has (Reader Reference) sig m, Has S.Statement sig m) => Term -> m val
+eval0 :: (Has (Env addr) sig m, HasLabelled Store (Store addr val) sig m, Has (Dom val) sig m, Has (Reader Reference) sig m, Has S.Statement sig m, MonadFail m) => Term -> m val
 eval0 = fix eval
 
 eval
-  :: (Has (Env addr) sig m, HasLabelled Store (Store addr val) sig m, Has (Dom val) sig m, Has (Reader Reference) sig m, Has S.Statement sig m)
+  :: (Has (Env addr) sig m, HasLabelled Store (Store addr val) sig m, Has (Dom val) sig m, Has (Reader Reference) sig m, Has S.Statement sig m, MonadFail m)
   => (Term -> m val)
   -> (Term -> m val)
 eval eval = \case
-  T.Var n  -> lookupEnv n >>= maybe (dvar n) fetch
-  T.Term s -> case s of
-    Noop      -> dunit
-    Iff c t e -> do
-      c' <- eval c
-      dif c' (eval t) (eval e)
-    Bool b    -> dbool b
-    String s  -> dstring s
-    Throw e   -> eval e >>= ddie
-    Let n v b -> do
-      v' <- eval v
-      let' n v' (eval b)
-    t :>> u   -> do
-      t' <- eval t
-      u' <- eval u
-      t' >>> u'
-    Import ns -> S.simport ns >> dunit
-    Function n ps b -> letrec n (dabs ps (foldr (\ (p, a) m -> let' p a m) (eval b) . zip ps))
-    Call f as -> do
-      f' <- eval f
-      as' <- traverse eval as
-      dapp f' as'
-    Locate s t -> local (setSpan s) (eval t)
+  Module (Py.Module ss) -> suite ss
+  Statement (Py.Import is sp) -> setSpan sp $ do
+    for_ is $ \ Py.ImportItem{ Py.import_item_name = ns } -> case nonEmpty ns of
+      Nothing -> pure ()
+      Just ss -> S.simport (pack . Py.ident_string <$> ss)
+    dunit
+  Statement (Py.Pass sp) -> setSpan sp dunit
+  Statement (Py.Conditional cts e sp) -> setSpan sp $ foldr (\ (c, t) e -> do
+    c' <- eval (Expr c)
+    dif c' (suite t) e) (suite e) cts
+  Statement (Py.Raise (Py.RaiseV3 e) sp) -> setSpan sp $ case e of
+    Just (e, _) -> eval (Expr e) >>= ddie -- FIXME: from clause
+    Nothing     -> dunit >>= ddie
+    -- FIXME: RaiseV2
+    -- FIXME: whatever the second field is
+  Statement (Py.StmtExpr e sp) -> setSpan sp (eval (Expr e))
+  Statement (Py.Fun n ps _r ss sp) -> let ps' = mapMaybe (\case { Py.Param n _ _ _ -> Just (ident n) ; _ -> Nothing }) ps in setSpan sp $ letrec (ident n) (dabs ps' (foldr (\ (p, a) m -> let' p a m) (suite ss) . zip ps'))
+  Expr (Py.Var n sp) -> setSpan sp $ let n' = ident n in lookupEnv n' >>= maybe (dvar n') fetch
+  Expr (Py.Bool b sp) -> setSpan sp $ dbool b
+  Expr (Py.Strings ss sp) -> setSpan sp $ dstring (pack (mconcat ss))
+  Expr (Py.Call f as sp) -> setSpan sp $ do
+    f' <- eval (Expr f)
+    as' <- traverse eval (mapMaybe (\case { Py.ArgExpr e _ -> Just (Expr e) ; _ -> Nothing }) as)
+    -- FIXME: support keyword args &c.
+    dapp f' as'
+  _ -> fail "TBD"
   where
-  setSpan s r = r{ refSpan = s }
+  setSpan s = case fromSpan s of
+    Just s -> local (\ r -> r{ refSpan = s })
+    _      -> id
+  fromSpan Py.SpanEmpty                     = Nothing
+  fromSpan (Py.SpanPoint _ l c)             = Just (point (Pos l c))
+  fromSpan (Py.SpanCoLinear _ l c1 c2)      = Just (Span (Pos l c1) (Pos l c2))
+  fromSpan (Py.SpanMultiLine _ l1 l2 c1 c2) = Just (Span (Pos l1 c1) (Pos l2 c2))
+  suite []     = dunit
+  suite (s:ss) = do
+    s' <- eval (Statement s)
+    foldM (\ into each -> do
+      each' <- eval (Statement each)
+      into >>> each') s' ss
+  ident = name . pack . Py.ident_string
+
+
+-- Parsing
+
+parse :: FilePath -> IO Term
+parse path = do
+  src <- readFile path
+  case parseModule src (takeBaseName path) of
+    Left err                -> fail (show err)
+    Right (Py.Module ss, _) -> pure (Module (Py.Module ss))

semantic-ast/semantic-ast.cabal

@@ -76,7 +76,7 @@ library
     , semantic-source      ^>= 0.2
     , tasty                ^>= 1.2.3
     , tasty-hunit          ^>= 0.10.0.2
-    , template-haskell      >= 2.15 && < 2.19
+    , template-haskell      >= 2.15 && < 2.22
     , text                 ^>= 1.2.3.1
     , tree-sitter          ^>= 0.9.0.0
     , unordered-containers ^>= 0.2.10

semantic-codeql/semantic-codeql.cabal

@@ -59,7 +59,7 @@ library
     , semantic-proto ^>= 0
     , semantic-source ^>= 0.2
     , semantic-tags ^>= 0.0
-    , template-haskell >= 2.15 && < 2.19
+    , template-haskell >= 2.15 && < 2.22
     , text ^>= 1.2.3
     , tree-sitter ^>= 0.9
     , tree-sitter-ql ^>= 0.1.0.2

semantic-go/semantic-go.cabal

@@ -59,7 +59,7 @@ library
     , semantic-proto ^>= 0
     , semantic-source ^>= 0.2
     , semantic-tags ^>= 0.0
-    , template-haskell  >= 2.15 && < 2.19
+    , template-haskell  >= 2.15 && < 2.22
     , text ^>= 1.2.3
     , tree-sitter ^>= 0.9
     , tree-sitter-go ^>= 0.5.0.1

semantic-java/semantic-java.cabal

@@ -59,7 +59,7 @@ library
     , semantic-proto ^>= 0
     , semantic-source ^>= 0.2
     , semantic-tags ^>= 0.0
-    , template-haskell  >= 2.15 && < 2.19
+    , template-haskell  >= 2.15 && < 2.22
     , text ^>= 1.2.3
     , tree-sitter ^>= 0.9
     , tree-sitter-java ^>= 0.7.0.1

semantic-json/semantic-json.cabal

@@ -56,7 +56,7 @@ library
     , base  >= 4.13 && < 5
     , semantic-ast
     , semantic-tags ^>= 0.0
-    , template-haskell  >= 2.15 && < 2.19
+    , template-haskell  >= 2.15 && < 2.22
     , text ^>= 1.2.3
     , tree-sitter ^>= 0.9
     , tree-sitter-json ^>= 0.7.0.1

semantic-php/semantic-php.cabal

@@ -57,7 +57,7 @@ library
     , semantic-proto ^>= 0
     , semantic-source ^>= 0.2
     , semantic-tags ^>= 0.0
-    , template-haskell  >= 2.15 && < 2.19
+    , template-haskell  >= 2.15 && < 2.22
     , text ^>= 1.2.3
     , tree-sitter ^>= 0.9
     , tree-sitter-php ^>= 0.5.0.0

semantic-python/semantic-python.cabal

@@ -65,7 +65,7 @@ library
     , semantic-proto ^>= 0
     , semantic-source ^>= 0.2
     , semantic-tags ^>= 0.0
-    , template-haskell  >= 2.15 && < 2.19
+    , template-haskell  >= 2.15 && < 2.22
     , text ^>= 1.2.3
     , tree-sitter ^>= 0.9
     , tree-sitter-python ^>= 0.9.0.2

semantic-ruby/semantic-ruby.cabal

@@ -59,7 +59,7 @@ library
     , semantic-proto ^>= 0
     , semantic-source ^>= 0.2
     , semantic-tags ^>= 0.0
-    , template-haskell  >= 2.15 && < 2.19
+    , template-haskell  >= 2.15 && < 2.22
     , text ^>= 1.2.3
     , tree-sitter ^>= 0.9
     , tree-sitter-ruby ^>= 0.5.0.2