Safe Haskell	Safe-Inferred
Language	GHC2021

Database.LSMTree.Internal.CRC32C

Contents

Pure incremental checksum calculation
I/O with checksum calculation
Checksum files
Checksum checking
File format errors

Description

Functionalty related to CRC-32C (Castagnoli) checksums:

Support for calculating checksums while incrementally writing files.
Support for verifying checksums of files.
Support for a text file format listing file checksums.

Synopsis

newtype CRC32C = CRC32C {
- unCRC32C :: Word32
}
initialCRC32C :: CRC32C
updateCRC32C :: ByteString -> CRC32C -> CRC32C
hGetSomeCRC32C :: Monad m => HasFS m h -> Handle h -> Word64 -> CRC32C -> m (ByteString, CRC32C)
hGetExactlyCRC32C :: MonadThrow m => HasFS m h -> Handle h -> Word64 -> CRC32C -> m (ByteString, CRC32C)
hPutSomeCRC32C :: Monad m => HasFS m h -> Handle h -> ByteString -> CRC32C -> m (Word64, CRC32C)
hPutAllCRC32C :: forall m h. Monad m => HasFS m h -> Handle h -> ByteString -> CRC32C -> m (Word64, CRC32C)
hPutAllChunksCRC32C :: forall m h. Monad m => HasFS m h -> Handle h -> ByteString -> CRC32C -> m (Word64, CRC32C)
readFileCRC32C :: forall m h. MonadThrow m => HasFS m h -> FsPath -> m CRC32C
newtype ChunkSize = ChunkSize ByteCount
defaultChunkSize :: ChunkSize
hGetExactlyCRC32C_SBS :: forall m h. (MonadThrow m, PrimMonad m) => HasFS m h -> Handle h -> ByteCount -> CRC32C -> m (ShortByteString, CRC32C)
hGetAllCRC32C' :: forall m h. PrimMonad m => HasFS m h -> Handle h -> ChunkSize -> CRC32C -> m CRC32C
type ChecksumsFile = Map ChecksumsFileName CRC32C
newtype ChecksumsFileName = ChecksumsFileName {
- unChecksumsFileName :: ByteString
}
getChecksum :: MonadThrow m => FsPath -> ChecksumsFile -> ChecksumsFileName -> m CRC32C
readChecksumsFile :: MonadThrow m => HasFS m h -> FsPath -> m ChecksumsFile
writeChecksumsFile :: MonadThrow m => HasFS m h -> FsPath -> ChecksumsFile -> m ()
writeChecksumsFile' :: MonadThrow m => HasFS m h -> Handle h -> ChecksumsFile -> m ()
checkCRC :: forall m h. (MonadMask m, PrimMonad m) => HasFS m h -> HasBlockIO m h -> Bool -> CRC32C -> FsPath -> m ()
expectChecksum :: MonadThrow m => FsPath -> CRC32C -> CRC32C -> m ()
data FileFormat
- = FormatChecksumsFile
- | FormatBloomFilterFile
- | FormatIndexFile
- | FormatWriteBufferFile
- | FormatSnapshotMetaData
data FileCorruptedError
- = ErrFileFormatInvalid !FsPath !FileFormat !String
- | ErrFileChecksumMismatch !FsPath !Word32 !Word32
expectValidFile :: MonadThrow m => FsPath -> FileFormat -> Either String a -> m a

Documentation

newtype CRC32C Source #

Constructors

CRC32C
Fields unCRC32C :: Word32

Instances

Instances details

Show CRC32C Source #
Instance details Defined in Database.LSMTree.Internal.CRC32C Methods showsPrec :: Int -> CRC32C -> ShowS # show :: CRC32C -> String # showList :: [CRC32C] -> ShowS #
Eq CRC32C Source #
Instance details Defined in Database.LSMTree.Internal.CRC32C Methods (==) :: CRC32C -> CRC32C -> Bool # (/=) :: CRC32C -> CRC32C -> Bool #
Ord CRC32C Source #
Instance details Defined in Database.LSMTree.Internal.CRC32C Methods compare :: CRC32C -> CRC32C -> Ordering # (<) :: CRC32C -> CRC32C -> Bool # (<=) :: CRC32C -> CRC32C -> Bool # (>) :: CRC32C -> CRC32C -> Bool # (>=) :: CRC32C -> CRC32C -> Bool # max :: CRC32C -> CRC32C -> CRC32C # min :: CRC32C -> CRC32C -> CRC32C #
Prim CRC32C Source #
Instance details Defined in Database.LSMTree.Internal.CRC32C Methods sizeOfType# :: Proxy CRC32C -> Int# Source # sizeOf# :: CRC32C -> Int# Source # alignmentOfType# :: Proxy CRC32C -> Int# Source # alignment# :: CRC32C -> Int# Source # indexByteArray# :: ByteArray# -> Int# -> CRC32C Source # readByteArray# :: MutableByteArray# s -> Int# -> State# s -> (# State# s, CRC32C #) Source # writeByteArray# :: MutableByteArray# s -> Int# -> CRC32C -> State# s -> State# s Source # setByteArray# :: MutableByteArray# s -> Int# -> Int# -> CRC32C -> State# s -> State# s Source # indexOffAddr# :: Addr# -> Int# -> CRC32C Source # readOffAddr# :: Addr# -> Int# -> State# s -> (# State# s, CRC32C #) Source # writeOffAddr# :: Addr# -> Int# -> CRC32C -> State# s -> State# s Source # setOffAddr# :: Addr# -> Int# -> Int# -> CRC32C -> State# s -> State# s Source #

Pure incremental checksum calculation

initialCRC32C :: CRC32C Source #

updateCRC32C :: ByteString -> CRC32C -> CRC32C Source #

I/O with checksum calculation

hGetSomeCRC32C :: Monad m => HasFS m h -> Handle h -> Word64 -> CRC32C -> m (ByteString, CRC32C) Source #

hGetExactlyCRC32C :: MonadThrow m => HasFS m h -> Handle h -> Word64 -> CRC32C -> m (ByteString, CRC32C) Source #

This function ensures that exactly the requested number of bytes is read. If the file is too short, an FsError of type FsReachedEOF is thrown.

It attempts to read everything into a single strict chunk, which should almost always succeed. If it doesn't, multiple chunks are produced.

TODO: To reliably return a strict bytestring without additional copying, fs-api needs to support directly reading into a buffer, which is currently work in progress: https://github.com/input-output-hk/fs-sim/pull/46

hPutSomeCRC32C :: Monad m => HasFS m h -> Handle h -> ByteString -> CRC32C -> m (Word64, CRC32C) Source #

hPutAllCRC32C :: forall m h. Monad m => HasFS m h -> Handle h -> ByteString -> CRC32C -> m (Word64, CRC32C) Source #

This function makes sure that the whole ByteString is written.

hPutAllChunksCRC32C :: forall m h. Monad m => HasFS m h -> Handle h -> ByteString -> CRC32C -> m (Word64, CRC32C) Source #

This function makes sure that the whole lazy ByteString is written.

readFileCRC32C :: forall m h. MonadThrow m => HasFS m h -> FsPath -> m CRC32C Source #

newtype ChunkSize Source #

Constructors

ChunkSize ByteCount

defaultChunkSize :: ChunkSize Source #

hGetExactlyCRC32C_SBS Source #

Arguments

:: forall m h. (MonadThrow m, PrimMonad m)
=> HasFS m h
-> Handle h
-> ByteCount	Number of bytes to read
-> CRC32C
-> m (ShortByteString, CRC32C)

Reads exactly as many bytes as requested, returning a ShortByteString and updating a given CRC32C value.

If EOF is found before the requested number of bytes is read, an FsError exception is thrown.

The returned ShortByteString is backed by pinned memory.

hGetAllCRC32C' Source #

Arguments

:: forall m h. PrimMonad m
=> HasFS m h
-> Handle h
-> ChunkSize	Chunk size, must be larger than 0
-> CRC32C
-> m CRC32C

Reads all bytes, updating a given CRC32C value without returning the bytes.

Checksum files

We use .checksum files to help verify the integrity of on disk snapshots. Each .checksum file lists the CRC-32C (Castagnoli) of other files. For further details see doc/format-directory.md.

The file uses the BSD-style checksum format (e.g. as produced by tools like md5sum --tag), with the algorithm name "CRC32C". This format is text, one line per file, using hexedecimal for the 32bit output.

Checksum files are used for each LSM run, and for the snapshot metadata.

Typical examples are:

CRC32C (keyops) = fd040004
CRC32C (blobs) = 5a3b820c
CRC32C (filter) = 6653e178
CRC32C (index) = f4ec6724

Or

CRC32C (snapshot) = 87972d7f

type ChecksumsFile = Map ChecksumsFileName CRC32C Source #

newtype ChecksumsFileName Source #

File names must not include characters '(', ')' or 'n'.

Constructors

ChecksumsFileName
Fields unChecksumsFileName :: ByteString

Instances

Instances details

Show ChecksumsFileName Source #
Instance details Defined in Database.LSMTree.Internal.CRC32C Methods showsPrec :: Int -> ChecksumsFileName -> ShowS # show :: ChecksumsFileName -> String # showList :: [ChecksumsFileName] -> ShowS #
Eq ChecksumsFileName Source #
Instance details Defined in Database.LSMTree.Internal.CRC32C Methods (==) :: ChecksumsFileName -> ChecksumsFileName -> Bool # (/=) :: ChecksumsFileName -> ChecksumsFileName -> Bool #
Ord ChecksumsFileName Source #
Instance details Defined in Database.LSMTree.Internal.CRC32C Methods compare :: ChecksumsFileName -> ChecksumsFileName -> Ordering # (<) :: ChecksumsFileName -> ChecksumsFileName -> Bool # (<=) :: ChecksumsFileName -> ChecksumsFileName -> Bool # (>) :: ChecksumsFileName -> ChecksumsFileName -> Bool # (>=) :: ChecksumsFileName -> ChecksumsFileName -> Bool # max :: ChecksumsFileName -> ChecksumsFileName -> ChecksumsFileName # min :: ChecksumsFileName -> ChecksumsFileName -> ChecksumsFileName #

getChecksum :: MonadThrow m => FsPath -> ChecksumsFile -> ChecksumsFileName -> m CRC32C Source #

readChecksumsFile :: MonadThrow m => HasFS m h -> FsPath -> m ChecksumsFile Source #

writeChecksumsFile :: MonadThrow m => HasFS m h -> FsPath -> ChecksumsFile -> m () Source #

writeChecksumsFile' :: MonadThrow m => HasFS m h -> Handle h -> ChecksumsFile -> m () Source #

Checksum checking

checkCRC :: forall m h. (MonadMask m, PrimMonad m) => HasFS m h -> HasBlockIO m h -> Bool -> CRC32C -> FsPath -> m () Source #

Check the CRC32C checksum for a file.

If the boolean argument is True, all file data for this path is evicted from the page cache.

expectChecksum :: MonadThrow m => FsPath -> CRC32C -> CRC32C -> m () Source #

File format errors

data FileFormat Source #

Constructors

FormatChecksumsFile
FormatBloomFilterFile
FormatIndexFile
FormatWriteBufferFile
FormatSnapshotMetaData

Instances

Instances details

Show FileFormat Source #
Instance details Defined in Database.LSMTree.Internal.CRC32C Methods showsPrec :: Int -> FileFormat -> ShowS # show :: FileFormat -> String # showList :: [FileFormat] -> ShowS #
Eq FileFormat Source #
Instance details Defined in Database.LSMTree.Internal.CRC32C Methods (==) :: FileFormat -> FileFormat -> Bool # (/=) :: FileFormat -> FileFormat -> Bool #

data FileCorruptedError Source #

The file is corrupted.

Constructors

ErrFileFormatInvalid	The file fails to parse.
Fields !FsPath File. !FileFormat File format. !String Error message.
ErrFileChecksumMismatch	The file CRC32 checksum is invalid.
Fields !FsPath File. !Word32 Expected checksum. !Word32 Actual checksum.

Instances

Instances details

Exception FileCorruptedError Source #
Instance details Defined in Database.LSMTree.Internal.CRC32C Methods toException :: FileCorruptedError -> SomeException # fromException :: SomeException -> Maybe FileCorruptedError # displayException :: FileCorruptedError -> String #
Show FileCorruptedError Source #
Instance details Defined in Database.LSMTree.Internal.CRC32C Methods showsPrec :: Int -> FileCorruptedError -> ShowS # show :: FileCorruptedError -> String # showList :: [FileCorruptedError] -> ShowS #
Eq FileCorruptedError Source #
Instance details Defined in Database.LSMTree.Internal.CRC32C Methods (==) :: FileCorruptedError -> FileCorruptedError -> Bool # (/=) :: FileCorruptedError -> FileCorruptedError -> Bool #

expectValidFile :: MonadThrow m => FsPath -> FileFormat -> Either String a -> m a Source #