Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update to network 3.0 with backward compatibility for network <= 2.8 #98

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
dist/
cabal.sandbox.config
.cabal-sandbox/
.stack-work/
8 changes: 2 additions & 6 deletions Database/MongoDB/Connection.hs
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import Control.Applicative ((<$>))
#endif

import Control.Monad (forM_)
import Network (HostName, PortID(..), connectTo)
import System.IO.Unsafe (unsafePerformIO)
import System.Timeout (timeout)
import Text.ParserCombinators.Parsec (parse, many1, letter, digit, char, eof,
Expand All @@ -39,7 +38,7 @@ import qualified Data.List as List


import Control.Monad.Identity (runIdentity)
import Control.Monad.Error (throwError)
import Control.Monad.Except (throwError)
import Control.Concurrent.MVar.Lifted (MVar, newMVar, withMVar, modifyMVar,
readMVar)
import Data.Bson (Document, at, (=:))
Expand All @@ -48,6 +47,7 @@ import Data.Text (Text)
import qualified Data.Bson as B
import qualified Data.Text as T

import Database.MongoDB.Internal.Network (HostName, PortID(..), connectTo)
import Database.MongoDB.Internal.Protocol (Pipe, newPipe, close, isClosed)
import Database.MongoDB.Internal.Util (untilSuccess, liftIOE,
updateAssocs, shuffle, mergesortM)
Expand Down Expand Up @@ -79,11 +79,7 @@ showHostPort :: Host -> String
-- TODO: Distinguish Service and UnixSocket port
showHostPort (Host hostname port) = hostname ++ ":" ++ portname where
portname = case port of
Service s -> s
PortNumber p -> show p
#if !defined(mingw32_HOST_OS) && !defined(cygwin32_HOST_OS) && !defined(_WIN32)
UnixSocket s -> s
#endif

readHostPortM :: (Monad m) => String -> m Host
-- ^ Read string \"hostname:port\" as @Host hosthame (PortNumber port)@ or \"hostname\" as @host hostname@ (default port). Fail if string does not match either syntax.
Expand Down
38 changes: 19 additions & 19 deletions Database/MongoDB/GridFS.hs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import Control.Applicative((<$>))

import Control.Monad(when)
import Control.Monad.IO.Class
import Control.Monad.Trans(MonadTrans, lift)
import Control.Monad.Trans(lift)

import Data.Conduit
import Data.Digest.Pure.MD5
Expand Down Expand Up @@ -121,25 +121,25 @@ sourceFile file = yieldChunk 0 where

-- Used to keep data during writing
data FileWriter = FileWriter
{ fwChunkSize :: Int64
, fwBucket :: Bucket
, fwFilesId :: ObjectId
, fwChunkIndex :: Int
, fwSize :: Int64
, fwAcc :: L.ByteString
, fwMd5Context :: MD5Context
, fwMd5acc :: L.ByteString
{ _fwChunkSize :: Int64
, _fwBucket :: Bucket
, _fwFilesId :: ObjectId
, _fwChunkIndex :: Int
, _fwSize :: Int64
, _fwAcc :: L.ByteString
, _fwMd5Context :: MD5Context
, _fwMd5acc :: L.ByteString
}

-- Finalize file, calculating md5 digest, saving the last chunk, and creating the file in the bucket
finalizeFile :: (Monad m, MonadIO m) => Text -> FileWriter -> Action m File
finalizeFile filename (FileWriter chunkSize bucket files_id i size acc md5context md5acc) = do
let md5digest = finalizeMD5 md5context (L.toStrict md5acc)
when (L.length acc > 0) $ putChunk bucket files_id i acc
timestamp <- liftIO $ getCurrentTime
currentTimestamp <- liftIO $ getCurrentTime
let doc = [ "_id" =: files_id
, "length" =: size
, "uploadDate" =: timestamp
, "uploadDate" =: currentTimestamp
, "md5" =: show (md5digest)
, "chunkSize" =: chunkSize
, "filename" =: filename
Expand All @@ -149,13 +149,13 @@ finalizeFile filename (FileWriter chunkSize bucket files_id i size acc md5contex

-- finalize the remainder and return the MD5Digest.
finalizeMD5 :: MD5Context -> S.ByteString -> MD5Digest
finalizeMD5 ctx rest =
md5Finalize ctx2 (S.drop lu rest) -- can only handle max md5BlockSizeInBytes length
finalizeMD5 ctx remainder =
md5Finalize ctx2 (S.drop lu remainder) -- can only handle max md5BlockSizeInBytes length
where
l = S.length rest
l = S.length remainder
r = l `mod` md5BlockSizeInBytes
lu = l - r
ctx2 = md5Update ctx (S.take lu rest)
ctx2 = md5Update ctx (S.take lu remainder)

-- Write as many chunks as can be written from the file writer
writeChunks :: (Monad m, MonadIO m) => FileWriter -> L.ByteString -> Action m FileWriter
Expand All @@ -167,16 +167,16 @@ writeChunks (FileWriter chunkSize bucket files_id i size acc md5context md5acc)
if (L.length md5acc_temp < md5BlockLength)
then (md5context, md5acc_temp)
else let numBlocks = L.length md5acc_temp `div` md5BlockLength
(current, rest) = L.splitAt (md5BlockLength * numBlocks) md5acc_temp
in (md5Update md5context (L.toStrict current), rest)
(current, remainder) = L.splitAt (md5BlockLength * numBlocks) md5acc_temp
in (md5Update md5context (L.toStrict current), remainder)
-- Update chunks
let size' = (size + L.length chunk)
let acc_temp = (acc `L.append` chunk)
if (L.length acc_temp < chunkSize)
then return (FileWriter chunkSize bucket files_id i size' acc_temp md5context' md5acc')
else do
let (chunk, acc') = L.splitAt chunkSize acc_temp
putChunk bucket files_id i chunk
let (newChunk, acc') = L.splitAt chunkSize acc_temp
putChunk bucket files_id i newChunk
writeChunks (FileWriter chunkSize bucket files_id (i+1) size' acc' md5context' md5acc') L.empty

sinkFile :: (Monad m, MonadIO m) => Bucket -> Text -> Consumer S.ByteString (Action m) File
Expand Down
52 changes: 52 additions & 0 deletions Database/MongoDB/Internal/Network.hs
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
-- | Compatibility layer for network package, including newtype 'PortID'
{-# LANGUAGE CPP, GeneralizedNewtypeDeriving #-}

module Database.MongoDB.Internal.Network (PortID(..), N.HostName, connectTo) where


#if !MIN_VERSION_network(2, 9, 0)

import qualified Network as N
import System.IO (Handle)

#else

import Control.Exception (bracketOnError)
import Network.BSD as BSD
import qualified Network.Socket as N
import System.IO (Handle, IOMode(ReadWriteMode))

#endif


-- | Wraps network's 'PortNumber'
-- Used to ease compatibility between older and newer network versions.
newtype PortID = PortNumber N.PortNumber deriving (Enum, Eq, Integral, Num, Ord, Read, Real, Show)


#if !MIN_VERSION_network(2, 9, 0)

-- Unwrap our newtype and use network's PortID and connectTo
connectTo :: N.HostName -- Hostname
-> PortID -- Port Identifier
-> IO Handle -- Connected Socket
connectTo hostname (PortNumber port) = N.connectTo hostname (N.PortNumber port)

#else

-- Copied implementation from network 2.8's 'connectTo', but using our 'PortID' newtype.
-- https://github.com/haskell/network/blob/e73f0b96c9da924fe83f3c73488f7e69f712755f/Network.hs#L120-L129
connectTo :: N.HostName -- Hostname
-> PortID -- Port Identifier
-> IO Handle -- Connected Socket
connectTo hostname (PortNumber port) = do
proto <- BSD.getProtocolNumber "tcp"
bracketOnError
(N.socket N.AF_INET N.Stream proto)
(N.close) -- only done if there's an error
(\sock -> do
he <- BSD.getHostByName hostname
N.connect sock (N.SockAddrInet port (hostAddress he))
N.socketToHandle sock ReadWriteMode
)
#endif
20 changes: 7 additions & 13 deletions Database/MongoDB/Internal/Util.hs
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
-- | Miscellaneous general functions and Show, Eq, and Ord instances for PortID
-- | Miscellaneous general functions

{-# LANGUAGE FlexibleInstances, UndecidableInstances, StandaloneDeriving #-}
{-# LANGUAGE CPP #-}
-- PortID instances
{-# OPTIONS_GHC -fno-warn-orphans #-}

module Database.MongoDB.Internal.Util where

Expand All @@ -14,26 +12,19 @@ import Control.Exception (handle, throwIO, Exception)
import Control.Monad (liftM, liftM2)
import Data.Bits (Bits, (.|.))
import Data.Word (Word8)
import Network (PortID(..))
import Numeric (showHex)
import System.Random (newStdGen)
import System.Random.Shuffle (shuffle')

import qualified Data.ByteString as S

import Control.Monad.Error (MonadError(..), Error(..))
import Control.Monad.Except (MonadError(..))
import Control.Monad.Trans (MonadIO, liftIO)
import Data.Bson
import Data.Text (Text)

import qualified Data.Text as T

#if !MIN_VERSION_network(2, 4, 1)
deriving instance Show PortID
deriving instance Eq PortID
#endif
deriving instance Ord PortID

-- | A monadic sort implementation derived from the non-monadic one in ghc's Prelude
mergesortM :: Monad m => (a -> a -> m Ordering) -> [a] -> m [a]
mergesortM cmp = mergesortM' cmp . map wrap
Expand Down Expand Up @@ -69,9 +60,12 @@ loop :: Monad m => m (Maybe a) -> m [a]
-- ^ Repeatedy execute action, collecting results, until it returns Nothing
loop act = act >>= maybe (return []) (\a -> (a :) `liftM` loop act)

untilSuccess :: (MonadError e m, Error e) => (a -> m b) -> [a] -> m b
untilSuccess :: (MonadError e m) => (a -> m b) -> [a] -> m b
-- ^ Apply action to elements one at a time until one succeeds. Throw last error if all fail. Throw 'strMsg' error if list is empty.
untilSuccess = untilSuccess' (strMsg "empty untilSuccess")
untilSuccess = untilSuccess' (error "empty untilSuccess")
-- Use 'error' copying behavior in removed 'Control.Monad.Error.Error' instance:
-- instance Error Failure where strMsg = error
-- 'fail' is treated the same as a programming 'error'. In other words, don't use it.

untilSuccess' :: (MonadError e m) => e -> (a -> m b) -> [a] -> m b
-- ^ Apply action to elements one at a time until one succeeds. Throw last error if all fail. Throw given error if list is empty
Expand Down
4 changes: 0 additions & 4 deletions Database/MongoDB/Query.hs
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ import Control.Concurrent.MVar.Lifted (MVar, addMVarFinalizer,
import Control.Applicative ((<$>))
import Control.Exception (catch)
import Control.Monad (when, void)
import Control.Monad.Error (Error(..))
import Control.Monad.Reader (MonadReader, ReaderT, runReaderT, ask, asks, local)
import Control.Monad.Trans (MonadIO, liftIO)
import Data.Binary.Put (runPut)
Expand Down Expand Up @@ -138,9 +137,6 @@ instance Exception Failure
type ErrorCode = Int
-- ^ Error code from getLastError or query failure

instance Error Failure where strMsg = error
-- ^ 'fail' is treated the same as a programming 'error'. In other words, don't use it.

-- | Type of reads and writes to perform
data AccessMode =
ReadStaleOk -- ^ Read-only action, reading stale data from a slave is OK.
Expand Down
4 changes: 2 additions & 2 deletions Database/MongoDB/Transport/Tls.hs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ import Database.MongoDB.Internal.Protocol (newPipeWith)
import Database.MongoDB.Transport (Transport(Transport))
import qualified Database.MongoDB.Transport as T
import System.IO.Error (mkIOError, eofErrorType)
import Network (connectTo, HostName, PortID)
import Database.MongoDB.Internal.Network (connectTo, HostName, PortID)
import qualified Network.TLS as TLS
import qualified Network.TLS.Extra.Cipher as TLS
import Database.MongoDB.Query (access, slaveOk, retrieveServerData)
Expand All @@ -50,7 +50,7 @@ connect host port = bracketOnError (connectTo host port) hClose $ \handle -> do

let params = (TLS.defaultParamsClient host "")
{ TLS.clientSupported = def
{ TLS.supportedCiphers = TLS.ciphersuite_all}
{ TLS.supportedCiphers = TLS.ciphersuite_default}
, TLS.clientHooks = def
{ TLS.onServerCertificate = \_ _ _ _ -> return []}
}
Expand Down
27 changes: 24 additions & 3 deletions mongoDB.cabal
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@ Build-type: Simple
Stability: alpha
Extra-Source-Files: CHANGELOG.md

-- Imitated from https://github.com/mongodb-haskell/bson/pull/18
Flag _old-network
description: Control whether to use <http://hackage.haskell.org/package/network-bsd network-bsd>
manual: False

Library
GHC-options: -Wall
default-language: Haskell2010
Expand All @@ -34,7 +39,6 @@ Library
, conduit-extra
, mtl >= 2
, cryptohash -any
, network -any
, parsec -any
, random -any
, random-shuffle -any
Expand All @@ -54,14 +58,23 @@ Library
, base64-bytestring >= 1.0.0.1
, nonce >= 1.0.5

if flag(_old-network)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@scott-fleischman Can you clarify for me who and how is supposed to set this flag?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They are set using the cabal configuration flags as described here.

There are a couple choices involved here that I basically deferred to the choices made in the bson PR.

  1. The name of the flag. _old-network is pretty cryptic. I just copied it. If flags can contain numbers it might be better to name it something like use-network2 or something like that.
  2. Default value. Again I made it False by default but maybe until network-3.* gets into stackage, maybe the default should be true? Then when stackage includes network 3 by default in the newest stackage lts then mongoDB could ship a breaking change which changes the flag to true, so it defaults to using network3.

I don't believe the .cabal files themselves allow logic like conditional statements based on a library dependency version. In our case we would need to include a new library (network-bsd) if network is >=3.0, so that might not be possible without configure flags.

In the end, I am happy to consider alternatives to the approach taken in this PR with regards to cabal configuration. But as far as I can tell that is going to involve some user choice of which network to use.

You can see how I set them in the stack files, for example:
https://github.com/plow-technologies/mongodb/blob/f84cc035179198ed63a5e247aa7c95f61ce7a295/stack-ghc80.yaml#L2-L4

The flag is omitted here, so it defaults to using the network 3.0+ and network-bsd dependencies.
https://github.com/plow-technologies/mongodb/blob/network-deprecations/stack-ghc86-network3.yaml

Copy link
Contributor Author

@scott-fleischman scott-fleischman Jun 3, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe manual: False might be useful here though and would work out-of-the-box in both cases (network 2 or network 3). https://www.haskell.org/cabal/users-guide/developing-packages.html#pkg-field-flag-manual

By default, Cabal will first try to satisfy dependencies with the default flag value and then, if that is not possible, with the negated value. However, if the flag is manual, then the default value (which can be overridden by commandline flags) will be used.

Copy link
Contributor Author

@scott-fleischman scott-fleischman Jun 3, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It doesn't seem to work for stack if I remove the explicit flag set. I get errors in the stack configure step.

cabal (cabal-install version 2.4.0.0) itself seems ok with it though. I was able to build using ghc 8.2.2 and it picked the old network configurations even though the _old-network flag defaults to False.

Copy link
Contributor Author

@scott-fleischman scott-fleischman Jun 3, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So all in all, it seems to me the current approach seems pretty good given the various options I am aware of.

Stack users will have to set the flag explicitly for network <=2.8.
Cabal users should work fine as-is. (I'm not sure how to test against the newer network because I'm not sure how to get cabal to know about the bson PR, which it would need to for network3.)

So with defaulting the flag to false, stack users of network 2.8 will have to explicitly set it to true (or explicitly include network 3). When the stackage lts includes network 3, then can remove the explicit flag from their configuration.

Cabal users should work fine in both cases. When network 3 is available for their configuration (needs bson available that works with network 3), since the _old-network flag default to True, it will try the network 3 config first, so it will use that. Else it default to the old network configuration and builds fine. I tested with ghc 8.2.2 and ghc 8.6.5 with cabal 2.4.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Here's a reference to the conditional options in a cabal file: https://www.haskell.org/cabal/users-guide/developing-packages.html#conditions

-- "Network.BSD" is only available in network < 2.9
build-depends: network < 2.9
else
-- "Network.BSD" has been moved into its own package `network-bsd`
build-depends: network >= 3.0
, network-bsd >= 2.7 && < 2.9

Exposed-modules: Database.MongoDB
Database.MongoDB.Admin
Database.MongoDB.Connection
Database.MongoDB.GridFS
Database.MongoDB.Query
Database.MongoDB.Transport
Database.MongoDB.Transport.Tls
Other-modules: Database.MongoDB.Internal.Protocol
Other-modules: Database.MongoDB.Internal.Network
Database.MongoDB.Internal.Protocol
Database.MongoDB.Internal.Util

Source-repository head
Expand Down Expand Up @@ -105,7 +118,6 @@ Benchmark bench
, containers -any
, mtl >= 2
, cryptohash -any
, network -any
, nonce >= 1.0.5
, stm
, parsec -any
Expand All @@ -116,5 +128,14 @@ Benchmark bench
, transformers-base >= 0.4.1
, hashtables >= 1.1.2.0
, criterion

if flag(_old-network)
-- "Network.BSD" is only available in network < 2.9
build-depends: network < 2.9
else
-- "Network.BSD" has been moved into its own package `network-bsd`
build-depends: network >= 3.0
, network-bsd >= 2.7 && < 2.9

default-language: Haskell2010
default-extensions: OverloadedStrings
4 changes: 4 additions & 0 deletions stack-ghc80.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
resolver: lts-9.21
flags:
mongoDB:
_old-network: true
4 changes: 4 additions & 0 deletions stack-ghc82.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
resolver: lts-11.22
flags:
mongoDB:
_old-network: true
4 changes: 4 additions & 0 deletions stack-ghc84.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
resolver: lts-12.26
flags:
mongoDB:
_old-network: true
6 changes: 6 additions & 0 deletions stack-ghc86-network3.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
resolver: lts-13.23
extra-deps:
- git: [email protected]:hvr/bson.git # https://github.com/mongodb-haskell/bson/pull/18
commit: 2fc8d04120c0758201762b8e22254aeb6d574f41
- network-bsd-2.8.1.0
- network-3.1.0.0
4 changes: 4 additions & 0 deletions stack-ghc86.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
resolver: lts-13.23
flags:
mongoDB:
_old-network: true