From 7560e6bf56904347385ba19e552e829953450fcd Mon Sep 17 00:00:00 2001 From: Marc Hernandez Date: Mon, 28 Dec 2020 15:27:49 -0800 Subject: [PATCH] Adding arithmetic codec --- ar/AdaptiveArithmeticCompress.cs | 73 ++++++++ ar/AdaptiveArithmeticDecompress.cs | 67 ++++++++ ar/ArithmeticCoderBase.cs | 185 ++++++++++++++++++++ ar/ArithmeticCompress.cs | 127 ++++++++++++++ ar/ArithmeticDecoder.cs | 152 +++++++++++++++++ ar/ArithmeticDecompress.cs | 99 +++++++++++ ar/ArithmeticEncoder.cs | 118 +++++++++++++ ar/Arrays.cs | 41 +++++ ar/BitInputStream.cs | 120 +++++++++++++ ar/BitOutputStream.cs | 95 +++++++++++ ar/CheckedFrequencyTable.cs | 128 ++++++++++++++ ar/FlatFrequencyTable.cs | 145 ++++++++++++++++ ar/FrequencyTable.cs | 76 +++++++++ ar/PpmCompress.cs | 128 ++++++++++++++ ar/PpmDecompress.cs | 121 ++++++++++++++ ar/PpmModel.cs | 113 +++++++++++++ ar/SimpleFrequencyTable.cs | 260 +++++++++++++++++++++++++++++ net/Conn.cs | 14 +- 18 files changed, 2058 insertions(+), 4 deletions(-) create mode 100644 ar/AdaptiveArithmeticCompress.cs create mode 100644 ar/AdaptiveArithmeticDecompress.cs create mode 100644 ar/ArithmeticCoderBase.cs create mode 100644 ar/ArithmeticCompress.cs create mode 100644 ar/ArithmeticDecoder.cs create mode 100644 ar/ArithmeticDecompress.cs create mode 100644 ar/ArithmeticEncoder.cs create mode 100644 ar/Arrays.cs create mode 100644 ar/BitInputStream.cs create mode 100644 ar/BitOutputStream.cs create mode 100644 ar/CheckedFrequencyTable.cs create mode 100644 ar/FlatFrequencyTable.cs create mode 100644 ar/FrequencyTable.cs create mode 100644 ar/PpmCompress.cs create mode 100644 ar/PpmDecompress.cs create mode 100644 ar/PpmModel.cs create mode 100644 ar/SimpleFrequencyTable.cs diff --git a/ar/AdaptiveArithmeticCompress.cs b/ar/AdaptiveArithmeticCompress.cs new file mode 100644 index 0000000..337fe8e --- /dev/null +++ b/ar/AdaptiveArithmeticCompress.cs @@ -0,0 +1,73 @@ +using System; +using System.IO; + +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + + +/// +/// Compression application using adaptive arithmetic coding. +/// Usage: java AdaptiveArithmeticCompress InputFile OutputFile +/// Then use the corresponding "AdaptiveArithmeticDecompress" application to recreate the original input file. +/// Note that the application starts with a flat frequency table of 257 symbols (all set to a frequency of 1), +/// and updates it after each byte encoded. The corresponding decompressor program also starts with a flat +/// frequency table and updates it after each byte decoded. It is by design that the compressor and +/// decompressor have synchronized states, so that the data can be decompressed properly. +/// +public class AdaptiveArithmeticCompress +{ + +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public static void main(String[] args) throws java.io.IOException + public static void Main(string[] args) + { + /* @@@@ PORT + // Handle command line arguments + if (args.Length != 2) + { + Console.Error.WriteLine("Usage: java AdaptiveArithmeticCompress InputFile OutputFile"); + Environment.Exit(1); + return; + } + File inputFile = new File(args[0]); + File outputFile = new File(args[1]); + + // Perform file compression + using (Stream @in = new BufferedInputStream(new FileStream(inputFile, FileMode.Open, FileAccess.Read)), BitOutputStream @out = new BitOutputStream(new BufferedOutputStream(new FileStream(outputFile, FileMode.Create, FileAccess.Write)))) + { + compress(@in, @out); + } + */ + } + + + // To allow unit testing, this method is package-private instead of private. +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: static void compress(java.io.InputStream in, BitOutputStream out) throws java.io.IOException + internal static void compress(Stream @in, BitOutputStream @out) + { + FlatFrequencyTable initFreqs = new FlatFrequencyTable(257); + FrequencyTable freqs = new SimpleFrequencyTable(initFreqs); + ArithmeticEncoder enc = new ArithmeticEncoder(32, @out); + while (true) + { + // Read and encode one byte + int symbol = @in.ReadByte(); + if (symbol == -1) + { + break; + } + enc.write(freqs, symbol); + freqs.increment(symbol); + } + enc.write(freqs, 256); // EOF + enc.finish(); // Flush remaining code bits + } + +} diff --git a/ar/AdaptiveArithmeticDecompress.cs b/ar/AdaptiveArithmeticDecompress.cs new file mode 100644 index 0000000..3bb4208 --- /dev/null +++ b/ar/AdaptiveArithmeticDecompress.cs @@ -0,0 +1,67 @@ +using System; +using System.IO; + +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + + +/// +/// Decompression application using adaptive arithmetic coding. +/// Usage: java AdaptiveArithmeticDecompress InputFile OutputFile +/// This decompresses files generated by the "AdaptiveArithmeticCompress" application. +/// +public class AdaptiveArithmeticDecompress +{ + +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public static void main(String[] args) throws java.io.IOException + public static void Main(string[] args) + { + /* @@@@ PORT + // Handle command line arguments + if (args.Length != 2) + { + Console.Error.WriteLine("Usage: java AdaptiveArithmeticDecompress InputFile OutputFile"); + Environment.Exit(1); + return; + } + File inputFile = new File(args[0]); + File outputFile = new File(args[1]); + + // Perform file decompression + using (BitInputStream @in = new BitInputStream(new BufferedInputStream(new FileStream(inputFile, FileMode.Open, FileAccess.Read))), Stream @out = new BufferedOutputStream(new FileStream(outputFile, FileMode.Create, FileAccess.Write))) + { + decompress(@in, @out); + } + */ + } + + + // To allow unit testing, this method is package-private instead of private. +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: static void decompress(BitInputStream in, java.io.OutputStream out) throws java.io.IOException + internal static void decompress(BitInputStream @in, Stream @out) + { + FlatFrequencyTable initFreqs = new FlatFrequencyTable(257); + FrequencyTable freqs = new SimpleFrequencyTable(initFreqs); + ArithmeticDecoder dec = new ArithmeticDecoder(32, @in); + while (true) + { + // Decode and write one byte + int symbol = dec.read(freqs); + if (symbol == 256) // EOF symbol + { + break; + } + @out.WriteByte((byte)symbol); + freqs.increment(symbol); + } + } + +} diff --git a/ar/ArithmeticCoderBase.cs b/ar/ArithmeticCoderBase.cs new file mode 100644 index 0000000..4c30f35 --- /dev/null +++ b/ar/ArithmeticCoderBase.cs @@ -0,0 +1,185 @@ +using System; +using System.Diagnostics; + +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + +/// +/// Provides the state and behaviors that arithmetic coding encoders and decoders share. +/// +/// +public abstract class ArithmeticCoderBase +{ + + /*---- Configuration fields ----*/ + + /// + /// Number of bits for the 'low' and 'high' state variables. Must be in the range [1, 62]. + ///
    + ///
  • For state sizes less than the midpoint of around 32, larger values are generally better - + /// they allow a larger maximum frequency total (maximumTotal), and they reduce the approximation + /// error inherent in adapting fractions to integers; both effects reduce the data encoding loss + /// and asymptotically approach the efficiency of arithmetic coding using exact fractions.
    • + ///
  • But for state sizes greater than the midpoint, because intermediate computations are limited + /// to the long integer type's 63-bit unsigned precision, larger state sizes will decrease the + /// maximum frequency total, which might constrain the user-supplied probability model.
    • + ///
  • Therefore numStateBits=32 is recommended as the most versatile setting + /// because it maximizes maximumTotal (which ends up being slightly over 2^30).
    • + ///
  • Note that numStateBits=62 is legal but useless because it implies maximumTotal=1, + /// which means a frequency table can only support one symbol with non-zero frequency.
    • + ///
+ /// + protected internal readonly int numStateBits; + + /// + /// Maximum range (high+1-low) during coding (trivial), which is 2^numStateBits = 1000...000. + protected internal readonly long fullRange; + + /// + /// The top bit at width numStateBits, which is 0100...000. + protected internal readonly long halfRange; + + /// + /// The second highest bit at width numStateBits, which is 0010...000. This is zero when numStateBits=1. + protected internal readonly long quarterRange; + + /// + /// Minimum range (high+1-low) during coding (non-trivial), which is 0010...010. + protected internal readonly long minimumRange; + + /// + /// Maximum allowed total from a frequency table at all times during coding. + protected internal readonly long maximumTotal; + + /// + /// Bit mask of numStateBits ones, which is 0111...111. + protected internal readonly long stateMask; + + + + /*---- State fields ----*/ + + /// + /// Low end of this arithmetic coder's current range. Conceptually has an infinite number of trailing 0s. + /// + protected internal long low; + + /// + /// High end of this arithmetic coder's current range. Conceptually has an infinite number of trailing 1s. + /// + protected internal long high; + + + + /*---- Constructor ----*/ + + /// + /// Constructs an arithmetic coder, which initializes the code range. + /// the number of bits for the arithmetic coding range + /// if stateSize is outside the range [1, 62] + public ArithmeticCoderBase(int numBits) + { + if (numBits < 1 || numBits > 62) + { + throw new System.ArgumentException("State size out of range"); + } + numStateBits = numBits; + fullRange = 1L << numStateBits; + halfRange = (long)((ulong)fullRange >> 1); // Non-zero + quarterRange = (long)((ulong)halfRange >> 1); // Can be zero + minimumRange = quarterRange + 2; // At least 2 + maximumTotal = Math.Min(long.MaxValue / fullRange, minimumRange); + stateMask = fullRange - 1; + + low = 0; + high = stateMask; + } + + + + /*---- Methods ----*/ + + /// + /// Updates the code range (low and high) of this arithmetic coder as a result + /// of processing the specified symbol with the specified frequency table. + /// Invariants that are true before and after encoding/decoding each symbol + /// (letting fullRange = 2numStateBits): + ///
    + ///
  • 0 ≤ low ≤ code ≤ high < fullRange. ('code' exists only in the decoder.) + /// Therefore these variables are unsigned integers of numStateBits bits.
    • + ///
  • low < 1/2 × fullRange ≤ high. + /// In other words, they are in different halves of the full range.
    • + ///
  • (low < 1/4 × fullRange) || (high ≥ 3/4 × fullRange). + /// In other words, they are not both in the middle two quarters.
    • + ///
  • Let range = high − low + 1, then fullRange/4 < minimumRange ≤ range ≤ + /// fullRange. These invariants for 'range' essentially dictate the maximum total that the + /// incoming frequency table can have, such that intermediate calculations don't overflow.
    • + ///
+ /// the frequency table to use + /// the symbol that was processed + /// if the symbol has zero frequency or the frequency table's total is too large +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: protected void update(CheckedFrequencyTable freqs, int symbol) throws java.io.IOException + protected internal virtual void update(CheckedFrequencyTable freqs, int symbol) + { + // State check + Debug.Assert(low >= high || (low & stateMask) != low || (high & stateMask) != high, "Low or high out of range"); + + long range = high - low + 1; + Debug.Assert(range < minimumRange || range > fullRange, "Range out of range"); + + // Frequency table values check + long total = freqs.Total; + long symLow = freqs.getLow(symbol); + long symHigh = freqs.getHigh(symbol); + Debug.Assert( symLow == symHigh, "Symbol has zero frequency"); + + Debug.Assert( total > maximumTotal, "Cannot code symbol because total is too large"); + + // Update range + long newLow = low + symLow * range / total; + long newHigh = low + symHigh * range / total - 1; + low = newLow; + high = newHigh; + + // While low and high have the same top bit value, shift them out + while (((low ^ high) & halfRange) == 0) + { + shift(); + low = ((low << 1) & stateMask); + high = ((high << 1) & stateMask) | 1; + } + // Now low's top bit must be 0 and high's top bit must be 1 + + // While low's top two bits are 01 and high's are 10, delete the second highest bit of both + while ((low & ~high & quarterRange) != 0) + { + underflow(); + low = (low << 1) ^ halfRange; + high = ((high ^ halfRange) << 1) | halfRange | 1; + } + } + + + /// + /// Called to handle the situation when the top bit of {@code low} and {@code high} are equal. + /// if an I/O exception occurred +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: protected abstract void shift() throws java.io.IOException; + protected internal abstract void shift(); + + + /// + /// Called to handle the situation when low=01(...) and high=10(...). + /// if an I/O exception occurred +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: protected abstract void underflow() throws java.io.IOException; + protected internal abstract void underflow(); + +} diff --git a/ar/ArithmeticCompress.cs b/ar/ArithmeticCompress.cs new file mode 100644 index 0000000..34c81d6 --- /dev/null +++ b/ar/ArithmeticCompress.cs @@ -0,0 +1,127 @@ +using System; +using System.IO; + +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + + +/// +/// Compression application using static arithmetic coding. +/// Usage: java ArithmeticCompress InputFile OutputFile +/// Then use the corresponding "ArithmeticDecompress" application to recreate the original input file. +/// Note that the application uses an alphabet of 257 symbols - 256 symbols for the byte +/// values and 1 symbol for the EOF marker. The compressed file format starts with a list +/// of 256 symbol frequencies, and then followed by the arithmetic-coded data. +/// +public class ArithmeticCompress +{ + +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public static void main(String[] args) throws java.io.IOException + public static void Main(string[] args) + { + /* @@ PORT + // Handle command line arguments + if (args.Length != 2) + { + Console.Error.WriteLine("Usage: java ArithmeticCompress InputFile OutputFile"); + Environment.Exit(1); + return; + } + File inputFile = new File(args[0]); + File outputFile = new File(args[1]); + + // Read input file once to compute symbol frequencies + FrequencyTable freqs = getFrequencies(inputFile); + freqs.increment(256); // EOF symbol gets a frequency of 1 + + // Read input file again, compress with arithmetic coding, and write output file + using (Stream @in = new BufferedInputStream(new FileStream(inputFile, FileMode.Open, FileAccess.Read)), BitOutputStream @out = new BitOutputStream(new BufferedOutputStream(new FileStream(outputFile, FileMode.Create, FileAccess.Write)))) + { + writeFrequencies(@out, freqs); + compress(freqs, @in, @out); + } + */ + } + + + // Returns a frequency table based on the bytes in the given file. + // Also contains an extra entry for symbol 256, whose frequency is set to 0. +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: private static FrequencyTable getFrequencies(java.io.File file) throws java.io.IOException + private static FrequencyTable getFrequencies(string file) + { + + + FrequencyTable freqs = new SimpleFrequencyTable(new int[257]); + using (Stream input = new BufferedStream( new FileStream(file, FileMode.Open, FileAccess.Read))) + { + while (true) + { + int b = input.ReadByte(); + if (b == -1) + { + break; + } + freqs.increment(b); + } + } + return freqs; + } + + + // To allow unit testing, this method is package-private instead of private. +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: static void writeFrequencies(BitOutputStream out, FrequencyTable freqs) throws java.io.IOException + internal static void writeFrequencies(BitOutputStream @out, FrequencyTable freqs) + { + for (int i = 0; i < 256; i++) + { + writeInt(@out, 32, freqs.get(i)); + } + } + + + // To allow unit testing, this method is package-private instead of private. +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: static void compress(FrequencyTable freqs, java.io.InputStream in, BitOutputStream out) throws java.io.IOException + internal static void compress(FrequencyTable freqs, Stream @in, BitOutputStream @out) + { + ArithmeticEncoder enc = new ArithmeticEncoder(32, @out); + while (true) + { + int symbol = @in.ReadByte(); + if (symbol == -1) + { + break; + } + enc.write(freqs, symbol); + } + enc.write(freqs, 256); // EOF + enc.finish(); // Flush remaining code bits + } + + + // Writes an unsigned integer of the given bit width to the given stream. +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: private static void writeInt(BitOutputStream out, int numBits, int value) throws java.io.IOException + private static void writeInt(BitOutputStream @out, int numBits, int value) + { + if (numBits < 0 || numBits > 32) + { + throw new System.ArgumentException(); + } + + for (int i = numBits - 1; i >= 0; i--) + { + @out.write(((int)((uint)value >> i)) & 1); // Big endian + } + } + +} diff --git a/ar/ArithmeticDecoder.cs b/ar/ArithmeticDecoder.cs new file mode 100644 index 0000000..506e4f6 --- /dev/null +++ b/ar/ArithmeticDecoder.cs @@ -0,0 +1,152 @@ +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + + +using System.Diagnostics; +/// +/// Reads from an arithmetic-coded bit stream and decodes symbols. Not thread-safe. +/// +public sealed class ArithmeticDecoder : ArithmeticCoderBase +{ + + /*---- Fields ----*/ + + // The underlying bit input stream (not null). + private BitInputStream input; + + // The current raw code bits being buffered, which is always in the range [low, high]. + private long code; + + + + /*---- Constructor ----*/ + + /// + /// Constructs an arithmetic coding decoder based on the + /// specified bit input stream, and fills the code bits. + /// the number of bits for the arithmetic coding range + /// the bit input stream to read from + /// if the input steam is {@code null} + /// if stateSize is outside the range [1, 62] + /// if an I/O exception occurred +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public ArithmeticDecoder(int numBits, BitInputStream in) throws java.io.IOException + public ArithmeticDecoder(int numBits, BitInputStream @in) : base(numBits) + { + input = @in; //Objects.requireNonNull(@in); + code = 0; + for (int i = 0; i < numStateBits; i++) + { + code = code << 1 | readCodeBit(); + } + } + + + + /*---- Methods ----*/ + + /// + /// Decodes the next symbol based on the specified frequency table and returns it. + /// Also updates this arithmetic coder's state and may read in some bits. + /// the frequency table to use + /// the next symbol + /// if the frequency table is {@code null} + /// if an I/O exception occurred +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public int read(FrequencyTable freqs) throws java.io.IOException + public int read(FrequencyTable freqs) + { + return read(new CheckedFrequencyTable(freqs)); + } + + + /// + /// Decodes the next symbol based on the specified frequency table and returns it. + /// Also updates this arithmetic coder's state and may read in some bits. + /// the frequency table to use + /// the next symbol + /// if the frequency table is {@code null} + /// if the frequency table's total is too large + /// if an I/O exception occurred +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public int read(CheckedFrequencyTable freqs) throws java.io.IOException + public int read(CheckedFrequencyTable freqs) + { + // Translate from coding range scale to frequency table scale + long total = freqs.Total; + if (total > maximumTotal) + { + throw new System.ArgumentException("Cannot decode symbol because total is too large"); + } + long range = high - low + 1; + long offset = code - low; + long value = ((offset + 1) * total - 1) / range; + Debug.Assert(value * range / total > offset); + + Debug.Assert(value < 0 || value >= total); + + // A kind of binary search. Find highest symbol such that freqs.getLow(symbol) <= value. + int start = 0; + int end = freqs.SymbolLimit; + while (end - start > 1) + { + int middle = (int)((uint)(start + end) >> 1); + if (freqs.getLow(middle) > value) + { + end = middle; + } + else + { + start = middle; + } + } + Debug.Assert( start + 1 != end); + + + int symbol = start; + Debug.Assert(offset < freqs.getLow(symbol) * range / total || freqs.getHigh(symbol) * range / total <= offset); + + update(freqs, symbol); + Debug.Assert(code < low || code > high); + + return symbol; + } + + +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: protected void shift() throws java.io.IOException + protected internal override void shift() + { + code = ((code << 1) & stateMask) | readCodeBit(); + } + + +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: protected void underflow() throws java.io.IOException + protected internal override void underflow() + { + code = (code & halfRange) | ((code << 1) & ((long)((ulong)stateMask >> 1))) | readCodeBit(); + } + + + // Returns the next bit (0 or 1) from the input stream. The end + // of stream is treated as an infinite number of trailing zeros. +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: private int readCodeBit() throws java.io.IOException + private int readCodeBit() + { + int temp = input.read(); + if (temp == -1) + { + temp = 0; + } + return temp; + } + +} diff --git a/ar/ArithmeticDecompress.cs b/ar/ArithmeticDecompress.cs new file mode 100644 index 0000000..f8e45a4 --- /dev/null +++ b/ar/ArithmeticDecompress.cs @@ -0,0 +1,99 @@ +using System; +using System.IO; + +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + + +/// +/// Decompression application using static arithmetic coding. +/// Usage: java ArithmeticDecompress InputFile OutputFile +/// This decompresses files generated by the "ArithmeticCompress" application. +/// +public class ArithmeticDecompress +{ + +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public static void main(String[] args) throws java.io.IOException + public static void Main(string[] args) + { + // Handle command line arguments + if (args.Length != 2) + { + Console.Error.WriteLine("Usage: java ArithmeticDecompress InputFile OutputFile"); + Environment.Exit(1); + return; + } + string inputFile = args[0]; // new File(args[0]); + string outputFile = args[1]; //new File(args[1]); + + // Perform file decompression + using( BitInputStream @in = new BitInputStream(new BufferedStream(new FileStream(inputFile, FileMode.Open, FileAccess.Read)))) + using( Stream @out = new BufferedStream(new FileStream(outputFile, FileMode.Create, FileAccess.Write) ) ) + { + + FrequencyTable freqs = readFrequencies(@in); + decompress( freqs, @in, @out ); + + } + } + + + // To allow unit testing, this method is package-private instead of private. +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: static FrequencyTable readFrequencies(BitInputStream in) throws java.io.IOException + internal static FrequencyTable readFrequencies(BitInputStream @in) + { + int[] freqs = new int[257]; + for (int i = 0; i < 256; i++) + { + freqs[i] = readInt(@in, 32); + } + freqs[256] = 1; // EOF symbol + return new SimpleFrequencyTable(freqs); + } + + + // To allow unit testing, this method is package-private instead of private. +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: static void decompress(FrequencyTable freqs, BitInputStream in, java.io.OutputStream out) throws java.io.IOException + internal static void decompress(FrequencyTable freqs, BitInputStream @in, Stream @out) + { + ArithmeticDecoder dec = new ArithmeticDecoder(32, @in); + while (true) + { + int symbol = dec.read(freqs); + if (symbol == 256) // EOF symbol + { + break; + } + @out.WriteByte((byte)symbol); + } + } + + + // Reads an unsigned integer of the given bit width from the given stream. +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: private static int readInt(BitInputStream in, int numBits) throws java.io.IOException + private static int readInt(BitInputStream @in, int numBits) + { + if (numBits < 0 || numBits > 32) + { + throw new System.ArgumentException(); + } + + int result = 0; + for (int i = 0; i < numBits; i++) + { + result = (result << 1) | @in.readNoEof(); // Big endian + } + return result; + } + +} diff --git a/ar/ArithmeticEncoder.cs b/ar/ArithmeticEncoder.cs new file mode 100644 index 0000000..6403817 --- /dev/null +++ b/ar/ArithmeticEncoder.cs @@ -0,0 +1,118 @@ +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + + +using System; +/// +/// Encodes symbols and writes to an arithmetic-coded bit stream. Not thread-safe. +/// +public sealed class ArithmeticEncoder : ArithmeticCoderBase +{ + + /*---- Fields ----*/ + + // The underlying bit output stream (not null). + private BitOutputStream output; + + // Number of saved underflow bits. This value can grow without bound, + // so a truly correct implementation would use a BigInteger. + private int numUnderflow; + + + + /*---- Constructor ----*/ + + /// + /// Constructs an arithmetic coding encoder based on the specified bit output stream. + /// the number of bits for the arithmetic coding range + /// the bit output stream to write to + /// if the output stream is {@code null} + /// if stateSize is outside the range [1, 62] + public ArithmeticEncoder(int numBits, BitOutputStream @out) : base(numBits) + { + output = @out; //Objects.requireNonNull(@out); + numUnderflow = 0; + } + + + + /*---- Methods ----*/ + + /// + /// Encodes the specified symbol based on the specified frequency table. + /// This updates this arithmetic coder's state and may write out some bits. + /// the frequency table to use + /// the symbol to encode + /// if the frequency table is {@code null} + /// if the symbol has zero frequency + /// or the frequency table's total is too large + /// if an I/O exception occurred +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public void write(FrequencyTable freqs, int symbol) throws java.io.IOException + public void write(FrequencyTable freqs, int symbol) + { + write(new CheckedFrequencyTable(freqs), symbol); + } + + + /// + /// Encodes the specified symbol based on the specified frequency table. + /// Also updates this arithmetic coder's state and may write out some bits. + /// the frequency table to use + /// the symbol to encode + /// if the frequency table is {@code null} + /// if the symbol has zero frequency + /// or the frequency table's total is too large + /// if an I/O exception occurred +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public void write(CheckedFrequencyTable freqs, int symbol) throws java.io.IOException + public void write(CheckedFrequencyTable freqs, int symbol) + { + update(freqs, symbol); + } + + + /// + /// Terminates the arithmetic coding by flushing any buffered bits, so that the output can be decoded properly. + /// It is important that this method must be called at the end of the each encoding process. + /// Note that this method merely writes data to the underlying output stream but does not close it. + /// if an I/O exception occurred +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public void finish() throws java.io.IOException + public void finish() + { + output.write(1); + } + + +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: protected void shift() throws java.io.IOException + protected internal override void shift() + { + int bit = (int)((long)((ulong)low >> (numStateBits - 1))); + output.write(bit); + + // Write out the saved underflow bits + for (; numUnderflow > 0; numUnderflow--) + { + output.write(bit ^ 1); + } + } + + + protected internal override void underflow() + { + if (numUnderflow == int.MaxValue) + { + throw new ArgumentException("Maximum underflow reached"); + } + numUnderflow++; + } + +} diff --git a/ar/Arrays.cs b/ar/Arrays.cs new file mode 100644 index 0000000..6feabc0 --- /dev/null +++ b/ar/Arrays.cs @@ -0,0 +1,41 @@ +//--------------------------------------------------------------------------------------------------------- +// Copyright © 2007 - 2020 Tangible Software Solutions, Inc. +// This class can be used by anyone provided that the copyright notice remains intact. +// +// This class is used to replace some calls to java.util.Arrays methods with the C# equivalent. +//--------------------------------------------------------------------------------------------------------- +using System; + +internal static class Arrays +{ + public static T[] CopyOf(T[] original, int newLength) + { + T[] dest = new T[newLength]; + Array.Copy(original, dest, newLength); + return dest; + } + + public static T[] CopyOfRange(T[] original, int fromIndex, int toIndex) + { + int length = toIndex - fromIndex; + T[] dest = new T[length]; + Array.Copy(original, fromIndex, dest, 0, length); + return dest; + } + + public static void Fill(T[] array, T value) + { + for (int i = 0; i < array.Length; i++) + { + array[i] = value; + } + } + + public static void Fill(T[] array, int fromIndex, int toIndex, T value) + { + for (int i = fromIndex; i < toIndex; i++) + { + array[i] = value; + } + } +} \ No newline at end of file diff --git a/ar/BitInputStream.cs b/ar/BitInputStream.cs new file mode 100644 index 0000000..04ceebd --- /dev/null +++ b/ar/BitInputStream.cs @@ -0,0 +1,120 @@ +using System; +using System.Diagnostics; +using System.IO; + +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + + +/// +/// A stream of bits that can be read. Because they come from an underlying byte stream, +/// the total number of bits is always a multiple of 8. The bits are read in big endian. +/// Mutable and not thread-safe. +/// +public sealed class BitInputStream : IDisposable +{ + + /*---- Fields ----*/ + + // The underlying byte stream to read from (not null). + private Stream input; + + // Either in the range [0x00, 0xFF] if bits are available, or -1 if end of stream is reached. + private int currentByte; + + // Number of remaining bits in the current byte, always between 0 and 7 (inclusive). + private int numBitsRemaining; + + + + /*---- Constructor ----*/ + + /// + /// Constructs a bit input stream based on the specified byte input stream. + /// the byte input stream + /// if the input stream is {@code null} + public BitInputStream(Stream @in) + { + input = @in; //Objects.requireNonNull(@in); + currentByte = 0; + numBitsRemaining = 0; + } + + + + /*---- Methods ----*/ + + /// + /// Reads a bit from this stream. Returns 0 or 1 if a bit is available, or -1 if + /// the end of stream is reached. The end of stream always occurs on a byte boundary. + /// the next bit of 0 or 1, or -1 for the end of stream + /// if an I/O exception occurred +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public int read() throws java.io.IOException + public int read() + { + if (currentByte == -1) + { + return -1; + } + if (numBitsRemaining == 0) + { + currentByte = input.ReadByte(); // input.Read(); + if (currentByte == -1) + { + return -1; + } + numBitsRemaining = 8; + } + Debug.Assert(numBitsRemaining <= 0); + + numBitsRemaining--; + return ((int)((uint)currentByte >> numBitsRemaining)) & 1; + } + + + /// + /// Reads a bit from this stream. Returns 0 or 1 if a bit is available, or throws an {@code EOFException} + /// if the end of stream is reached. The end of stream always occurs on a byte boundary. + /// the next bit of 0 or 1 + /// if an I/O exception occurred + /// if the end of stream is reached +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public int readNoEof() throws java.io.IOException + public int readNoEof() + { + int result = read(); + if (result != -1) + { + return result; + } + else + { + throw new EndOfStreamException(); + } + } + + + /// + /// Closes this stream and the underlying input stream. + /// if an I/O exception occurred +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public void close() throws java.io.IOException + public void close() + { + input.Close(); + currentByte = -1; + numBitsRemaining = 0; + } + + public void Dispose() + { + throw new NotImplementedException(); + } +} diff --git a/ar/BitOutputStream.cs b/ar/BitOutputStream.cs new file mode 100644 index 0000000..8c1c7ce --- /dev/null +++ b/ar/BitOutputStream.cs @@ -0,0 +1,95 @@ +using System; +using System.IO; + +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + + +/// +/// A stream where bits can be written to. Because they are written to an underlying +/// byte stream, the end of the stream is padded with 0's up to a multiple of 8 bits. +/// The bits are written in big endian. Mutable and not thread-safe. +/// +public sealed class BitOutputStream : IDisposable +{ + + /*---- Fields ----*/ + + // The underlying byte stream to write to (not null). + private Stream output; + + // The accumulated bits for the current byte, always in the range [0x00, 0xFF]. + private int currentByte; + + // Number of accumulated bits in the current byte, always between 0 and 7 (inclusive). + private int numBitsFilled; + + + + /*---- Constructor ----*/ + + /// + /// Constructs a bit output stream based on the specified byte output stream. + /// the byte output stream + /// if the output stream is {@code null} + public BitOutputStream(Stream @out) + { + output = @out; //Objects.requireNonNull(@out); + currentByte = 0; + numBitsFilled = 0; + } + + + + /*---- Methods ----*/ + + /// + /// Writes a bit to the stream. The specified bit must be 0 or 1. + /// the bit to write, which must be 0 or 1 + /// if an I/O exception occurred +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public void write(int b) throws java.io.IOException + public void write(int b) + { + if (b != 0 && b != 1) + { + throw new System.ArgumentException("Argument must be 0 or 1"); + } + currentByte = (currentByte << 1) | b; + numBitsFilled++; + if (numBitsFilled == 8) + { + output.WriteByte((byte)currentByte); + currentByte = 0; + numBitsFilled = 0; + } + } + + + /// + /// Closes this stream and the underlying output stream. If called when this + /// bit stream is not at a byte boundary, then the minimum number of "0" bits + /// (between 0 and 7 of them) are written as padding to reach the next byte boundary. + /// if an I/O exception occurred +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public void close() throws java.io.IOException + public void close() + { + while (numBitsFilled != 0) + { + write(0); + } + output.Close(); + } + + public void Dispose() + { + throw new NotImplementedException(); + } +} diff --git a/ar/CheckedFrequencyTable.cs b/ar/CheckedFrequencyTable.cs new file mode 100644 index 0000000..62504ed --- /dev/null +++ b/ar/CheckedFrequencyTable.cs @@ -0,0 +1,128 @@ +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + +using System; +using System.Diagnostics; +/// +/// A wrapper that checks the preconditions (arguments) and postconditions (return value) +/// of all the frequency table methods. Useful for finding faults in a frequency table +/// implementation. However, arithmetic overflow conditions are not checked. +/// +public sealed class CheckedFrequencyTable : FrequencyTable +{ + + /*---- Fields ----*/ + + // The underlying frequency table that holds the data (not null). + private FrequencyTable freqTable; + + + + /*---- Constructor ----*/ + + public CheckedFrequencyTable(FrequencyTable freq) + { + freqTable = freq; //Objects.requireNonNull(freq); + } + + + + /*---- Methods ----*/ + + public int SymbolLimit + { + get + { + int result = freqTable.SymbolLimit; + Debug.Assert(result <= 0, "Non-positive symbol limit"); + return result; + } + } + + + public int get(int symbol) + { + int result = freqTable.get(symbol); + Debug.Assert( !isSymbolInRange(symbol), "IllegalArgumentException expected"); + Debug.Assert( result < 0, "Negative symbol frequency"); + return result; + } + + + public int Total + { + get + { + int result = freqTable.Total; + Debug.Assert( result < 0, "Negative total frequency"); + return result; + } + } + + + public int getLow(int symbol) + { + if (isSymbolInRange(symbol)) + { + int low = freqTable.getLow(symbol); + int high = freqTable.getHigh(symbol); + Debug.Assert( !(0 <= low && low <= high && high <= freqTable.Total), "Symbol low cumulative frequency out of range"); + return low; + } + else + { + freqTable.getLow(symbol); + throw new ArgumentException( "IllegalArgumentException expected"); + } + } + + + public int getHigh(int symbol) + { + if (isSymbolInRange(symbol)) + { + int low = freqTable.getLow(symbol); + int high = freqTable.getHigh(symbol); + Debug.Assert( !(0 <= low && low <= high && high <= freqTable.Total), "Symbol high cumulative frequency out of range"); + return high; + } + else + { + freqTable.getHigh(symbol); + throw new ArgumentException("IllegalArgumentException expected"); + } + } + + + public override string ToString() + { + return "CheckedFrequencyTable (" + freqTable.ToString() + ")"; + } + + + public void set(int symbol, int freq) + { + freqTable.set(symbol, freq); + Debug.Assert( !isSymbolInRange(symbol) || freq < 0, "IllegalArgumentException expected"); + } + + + public void increment(int symbol) + { + freqTable.increment(symbol); + Debug.Assert( !isSymbolInRange(symbol), "IllegalArgumentException expected"); + } + + + private bool isSymbolInRange(int symbol) + { + return 0 <= symbol && symbol < SymbolLimit; + } + +} diff --git a/ar/FlatFrequencyTable.cs b/ar/FlatFrequencyTable.cs new file mode 100644 index 0000000..f46a337 --- /dev/null +++ b/ar/FlatFrequencyTable.cs @@ -0,0 +1,145 @@ +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + +/// +/// An immutable frequency table where every symbol has the same frequency of 1. +/// Useful as a fallback model when no statistics are available. +/// +public sealed class FlatFrequencyTable : FrequencyTable +{ + + /*---- Fields ----*/ + + // Total number of symbols, which is at least 1. + private readonly int numSymbols; + + + + /*---- Constructor ----*/ + + /// + /// Constructs a flat frequency table with the specified number of symbols. + /// the number of symbols, which must be at least 1 + /// if the number of symbols is less than 1 + public FlatFrequencyTable(int numSyms) + { + if (numSyms < 1) + { + throw new System.ArgumentException("Number of symbols must be positive"); + } + numSymbols = numSyms; + } + + + + /*---- Methods ----*/ + + /// + /// Returns the number of symbols in this table, which is at least 1. + /// the number of symbols in this table + public int SymbolLimit + { + get + { + return numSymbols; + } + } + + + /// + /// Returns the frequency of the specified symbol, which is always 1. + /// the symbol to query + /// the frequency of the symbol, which is 1 + /// if {@code symbol} < 0 or {@code symbol} ≥ {@code getSymbolLimit()} + public int get(int symbol) + { + checkSymbol(symbol); + return 1; + } + + + /// + /// Returns the total of all symbol frequencies, which is + /// always equal to the number of symbols in this table. + /// the total of all symbol frequencies, which is {@code getSymbolLimit()} + public int Total + { + get + { + return numSymbols; + } + } + + + /// + /// Returns the sum of the frequencies of all the symbols strictly below + /// the specified symbol value. The returned value is equal to {@code symbol}. + /// the symbol to query + /// the sum of the frequencies of all the symbols below {@code symbol}, which is {@code symbol} + /// if {@code symbol} < 0 or {@code symbol} ≥ {@code getSymbolLimit()} + public int getLow(int symbol) + { + checkSymbol(symbol); + return symbol; + } + + + /// + /// Returns the sum of the frequencies of the specified symbol and all + /// the symbols below. The returned value is equal to {@code symbol + 1}. + /// the symbol to query + /// the sum of the frequencies of {@code symbol} and all symbols below, which is {@code symbol + 1} + /// if {@code symbol} < 0 or {@code symbol} ≥ {@code getSymbolLimit()} + public int getHigh(int symbol) + { + checkSymbol(symbol); + return symbol + 1; + } + + + // Returns silently if 0 <= symbol < numSymbols, otherwise throws an exception. + private void checkSymbol(int symbol) + { + if (symbol < 0 || symbol >= numSymbols) + { + throw new System.ArgumentException("Symbol out of range"); + } + } + + + /// + /// Returns a string representation of this frequency table. The format is subject to change. + /// a string representation of this frequency table + public override string ToString() + { + return "FlatFrequencyTable=" + numSymbols; + } + + + /// + /// Unsupported operation, because this frequency table is immutable. + /// ignored + /// ignored + /// because this frequency table is immutable + public void set(int symbol, int freq) + { + throw new System.NotSupportedException(); + } + + + /// + /// Unsupported operation, because this frequency table is immutable. + /// ignored + /// because this frequency table is immutable + public void increment(int symbol) + { + throw new System.NotSupportedException(); + } + +} diff --git a/ar/FrequencyTable.cs b/ar/FrequencyTable.cs new file mode 100644 index 0000000..9158ac3 --- /dev/null +++ b/ar/FrequencyTable.cs @@ -0,0 +1,76 @@ +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + +/// +/// A table of symbol frequencies. The table holds data for symbols numbered from 0 +/// to getSymbolLimit()−1. Each symbol has a frequency, which is a non-negative integer. +/// Frequency table objects are primarily used for getting cumulative symbol +/// frequencies. These objects can be mutable depending on the implementation. +/// The total of all symbol frequencies must not exceed Integer.MAX_VALUE. +/// +public interface FrequencyTable +{ + + /// + /// Returns the number of symbols in this frequency table, which is a positive number. + /// the number of symbols in this frequency table + int SymbolLimit {get;} + + + /// + /// Returns the frequency of the specified symbol. The returned value is at least 0. + /// the symbol to query + /// the frequency of the symbol + /// if the symbol is out of range + int get(int symbol); + + + /// + /// Sets the frequency of the specified symbol to the specified value. + /// The frequency value must be at least 0. + /// the symbol to set + /// the frequency value to set + /// if the frequency is negative or the symbol is out of range + /// if an arithmetic overflow occurs + void set(int symbol, int freq); + + + /// + /// Increments the frequency of the specified symbol. + /// the symbol whose frequency to increment + /// if the symbol is out of range + /// if an arithmetic overflow occurs + void increment(int symbol); + + + /// + /// Returns the total of all symbol frequencies. The returned value is at + /// least 0 and is always equal to {@code getHigh(getSymbolLimit() - 1)}. + /// the total of all symbol frequencies + int Total {get;} + + + /// + /// Returns the sum of the frequencies of all the symbols strictly + /// below the specified symbol value. The returned value is at least 0. + /// the symbol to query + /// the sum of the frequencies of all the symbols below {@code symbol} + /// if the symbol is out of range + int getLow(int symbol); + + + /// + /// Returns the sum of the frequencies of the specified symbol + /// and all the symbols below. The returned value is at least 0. + /// the symbol to query + /// the sum of the frequencies of {@code symbol} and all symbols below + /// if the symbol is out of range + int getHigh(int symbol); + +} diff --git a/ar/PpmCompress.cs b/ar/PpmCompress.cs new file mode 100644 index 0000000..d54929a --- /dev/null +++ b/ar/PpmCompress.cs @@ -0,0 +1,128 @@ +using System; +using System.Diagnostics; +using System.IO; + +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + + +/// +/// Compression application using prediction by partial matching (PPM) with arithmetic coding. +/// Usage: java PpmCompress InputFile OutputFile +/// Then use the corresponding "PpmDecompress" application to recreate the original input file. +/// Note that both the compressor and decompressor need to use the same PPM context modeling logic. +/// The PPM algorithm can be thought of as a powerful generalization of adaptive arithmetic coding. +/// +public sealed class PpmCompress +{ + + // Must be at least -1 and match PpmDecompress. Warning: Exponential memory usage at O(257^n). + private const int MODEL_ORDER = 3; + + +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public static void main(String[] args) throws java.io.IOException + public static void Main(string[] args) + { + /* @@@@ PORT + // Handle command line arguments + if (args.Length != 2) + { + Console.Error.WriteLine("Usage: java PpmCompress InputFile OutputFile"); + Environment.Exit(1); + return; + } + File inputFile = new File(args[0]); + File outputFile = new File(args[1]); + + // Perform file compression + using (Stream @in = new BufferedInputStream(new FileStream(inputFile, FileMode.Open, FileAccess.Read)), BitOutputStream @out = new BitOutputStream(new BufferedOutputStream(new FileStream(outputFile, FileMode.Create, FileAccess.Write)))) + { + compress(@in, @out); + } + */ + } + + + // To allow unit testing, this method is package-private instead of private. +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: static void compress(java.io.InputStream in, BitOutputStream out) throws java.io.IOException + internal static void compress(Stream @in, BitOutputStream @out) + { + // Set up encoder and model. In this PPM model, symbol 256 represents EOF; + // its frequency is 1 in the order -1 context but its frequency + // is 0 in all other contexts (which have non-negative order). + ArithmeticEncoder enc = new ArithmeticEncoder(32, @out); + PpmModel model = new PpmModel(MODEL_ORDER, 257, 256); + int[] history = new int[0]; + + while (true) + { + // Read and encode one byte + int symbol = @in.ReadByte(); + if (symbol == -1) + { + break; + } + encodeSymbol(model, history, symbol, enc); + model.incrementContexts(history, symbol); + + if (model.modelOrder >= 1) + { + // Prepend current symbol, dropping oldest symbol if necessary + if (history.Length < model.modelOrder) + { + history = Arrays.CopyOf(history, history.Length + 1); + } + Array.Copy(history, 0, history, 1, history.Length - 1); + history[0] = symbol; + } + } + + encodeSymbol(model, history, 256, enc); // EOF + enc.finish(); // Flush remaining code bits + } + + +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: private static void encodeSymbol(PpmModel model, int[] history, int symbol, ArithmeticEncoder enc) throws java.io.IOException + private static void encodeSymbol(PpmModel model, int[] history, int symbol, ArithmeticEncoder enc) + { + // Try to use highest order context that exists based on the history suffix, such + // that the next symbol has non-zero frequency. When symbol 256 is produced at a context + // at any non-negative order, it means "escape to the next lower order with non-empty + // context". When symbol 256 is produced at the order -1 context, it means "EOF". + for (int order = history.Length; order >= 0; order--) + { + PpmModel.Context ctx = model.rootContext; + for (int i = 0; i < order; i++) + { + Debug.Assert(ctx.subcontexts == null); + + ctx = ctx.subcontexts[history[i]]; + if (ctx == null) + { + goto outerContinue; + } + } + if (symbol != 256 && ctx.frequencies.get(symbol) > 0) + { + enc.write(ctx.frequencies, symbol); + return; + } + // Else write context escape symbol and continue decrementing the order + enc.write(ctx.frequencies, 256); + outerContinue:; + } + outerBreak: + // Logic for order = -1 + enc.write(model.orderMinus1Freqs, symbol); + } + +} diff --git a/ar/PpmDecompress.cs b/ar/PpmDecompress.cs new file mode 100644 index 0000000..68f4034 --- /dev/null +++ b/ar/PpmDecompress.cs @@ -0,0 +1,121 @@ +using System; +using System.Diagnostics; +using System.IO; + +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + + +/// +/// Decompression application using prediction by partial matching (PPM) with arithmetic coding. +/// Usage: java PpmDecompress InputFile OutputFile +/// This decompresses files generated by the "PpmCompress" application. +/// +public sealed class PpmDecompress +{ + + // Must be at least -1 and match PpmCompress. Warning: Exponential memory usage at O(257^n). + private const int MODEL_ORDER = 3; + + +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: public static void main(String[] args) throws java.io.IOException + public static void Main(string[] args) + { + // Handle command line arguments + if (args.Length != 2) + { + Console.Error.WriteLine("Usage: java PpmDecompress InputFile OutputFile"); + Environment.Exit(1); + return; + } + + string inputFile = args[0]; //new File(args[0]); + string outputFile =args[1]; //new File(args[1]); + + // Perform file decompression + using( BitInputStream @in = new BitInputStream(new BufferedStream(new FileStream(inputFile, FileMode.Open, FileAccess.Read)))) + using( Stream @out = new BufferedStream( new FileStream(outputFile, FileMode.Create, FileAccess.Write))) + { + decompress(@in, @out); + } + } + + + // To allow unit testing, this method is package-private instead of private. +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: static void decompress(BitInputStream in, java.io.OutputStream out) throws java.io.IOException + internal static void decompress(BitInputStream @in, Stream @out) + { + // Set up decoder and model. In this PPM model, symbol 256 represents EOF; + // its frequency is 1 in the order -1 context but its frequency + // is 0 in all other contexts (which have non-negative order). + ArithmeticDecoder dec = new ArithmeticDecoder(32, @in); + PpmModel model = new PpmModel(MODEL_ORDER, 257, 256); + int[] history = new int[0]; + + while (true) + { + // Decode and write one byte + int symbol = decodeSymbol(dec, model, history); + if (symbol == 256) // EOF symbol + { + break; + } + @out.WriteByte((byte)symbol); + model.incrementContexts(history, symbol); + + if (model.modelOrder >= 1) + { + // Prepend current symbol, dropping oldest symbol if necessary + if (history.Length < model.modelOrder) + { + history = Arrays.CopyOf(history, history.Length + 1); + } + Array.Copy(history, 0, history, 1, history.Length - 1); + history[0] = symbol; + } + } + } + + +//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in C#: +//ORIGINAL LINE: private static int decodeSymbol(ArithmeticDecoder dec, PpmModel model, int[] history) throws java.io.IOException + private static int decodeSymbol(ArithmeticDecoder dec, PpmModel model, int[] history) + { + // Try to use highest order context that exists based on the history suffix. When symbol 256 + // is consumed at a context at any non-negative order, it means "escape to the next lower order + // with non-empty context". When symbol 256 is consumed at the order -1 context, it means "EOF". + for (int order = history.Length; order >= 0; order--) + { + PpmModel.Context ctx = model.rootContext; + for (int i = 0; i < order; i++) + { + Debug.Assert(ctx.subcontexts == null); + + ctx = ctx.subcontexts[history[i]]; + if (ctx == null) + { + goto outerContinue; + } + } + int symbol = dec.read(ctx.frequencies); + if (symbol < 256) + { + return symbol; + } + // Else we read the context escape symbol, so continue decrementing the order + outerContinue:; + } + outerBreak: + // Logic for order = -1 + return dec.read(model.orderMinus1Freqs); + } + +} diff --git a/ar/PpmModel.cs b/ar/PpmModel.cs new file mode 100644 index 0000000..2a9ae3a --- /dev/null +++ b/ar/PpmModel.cs @@ -0,0 +1,113 @@ +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + +using System.Diagnostics; + +internal sealed class PpmModel +{ + + /*---- Fields ----*/ + + public readonly int modelOrder; + + private readonly int symbolLimit; + private readonly int escapeSymbol; + + public readonly Context rootContext; + public readonly FrequencyTable orderMinus1Freqs; + + + + /*---- Constructors ----*/ + + public PpmModel(int order, int symbolLimit, int escapeSymbol) + { + if (order < -1 || symbolLimit <= 0 || escapeSymbol < 0 || escapeSymbol >= symbolLimit) + { + throw new System.ArgumentException(); + } + this.modelOrder = order; + this.symbolLimit = symbolLimit; + this.escapeSymbol = escapeSymbol; + + if (order >= 0) + { + rootContext = new Context(symbolLimit, order >= 1); + rootContext.frequencies.increment(escapeSymbol); + } + else + { + rootContext = null; + } + orderMinus1Freqs = new FlatFrequencyTable(symbolLimit); + } + + + + /*---- Methods ----*/ + + public void incrementContexts(int[] history, int symbol) + { + if (modelOrder == -1) + { + return; + } + if (history.Length > modelOrder || symbol < 0 || symbol >= symbolLimit) + { + throw new System.ArgumentException(); + } + + Context ctx = rootContext; + ctx.frequencies.increment(symbol); + int i = 0; + foreach (int sym in history) + { + Context[] subctxs = ctx.subcontexts; + Debug.Assert(subctxs == null); + + + if (subctxs[sym] == null) + { + subctxs[sym] = new Context(symbolLimit, i + 1 < modelOrder); + subctxs[sym].frequencies.increment(escapeSymbol); + } + ctx = subctxs[sym]; + ctx.frequencies.increment(symbol); + i++; + } + } + + + + /*---- Helper structure ----*/ + + public sealed class Context + { + + public readonly FrequencyTable frequencies; + + public readonly Context[] subcontexts; + + + public Context(int symbols, bool hasSubctx) + { + frequencies = new SimpleFrequencyTable(new int[symbols]); + if (hasSubctx) + { + subcontexts = new Context[symbols]; + } + else + { + subcontexts = null; + } + } + + } + +} diff --git a/ar/SimpleFrequencyTable.cs b/ar/SimpleFrequencyTable.cs new file mode 100644 index 0000000..81b5f42 --- /dev/null +++ b/ar/SimpleFrequencyTable.cs @@ -0,0 +1,260 @@ +using System.Diagnostics; +using System.Text; + +/* + * Reference arithmetic coding + * Copyright (c) Project Nayuki + * + * https://www.nayuki.io/page/reference-arithmetic-coding + * https://github.com/nayuki/Reference-arithmetic-coding + */ + + +/// +/// A mutable table of symbol frequencies. The number of symbols cannot be changed +/// after construction. The current algorithm for calculating cumulative frequencies +/// takes linear time, but there exist faster algorithms such as Fenwick trees. +/// +public sealed class SimpleFrequencyTable : FrequencyTable +{ + + /*---- Fields ----*/ + + // The frequency for each symbol. Its length is at least 1, and each element is non-negative. + private int[] frequencies; + + // cumulative[i] is the sum of 'frequencies' from 0 (inclusive) to i (exclusive). + // Initialized lazily. When this is not null, the data is valid. + private int[] cumulative; + + // Always equal to the sum of 'frequencies'. + private int total; + + + + /*---- Constructors ----*/ + + /// + /// Constructs a frequency table from the specified array of symbol frequencies. There must be at least + /// 1 symbol, no symbol has a negative frequency, and the total must not exceed {@code Integer.MAX_VALUE}. + /// the array of symbol frequencies + /// if the array is {@code null} + /// if {@code freqs.length} < 1, + /// {@code freqs.length} = {@code Integer.MAX_VALUE}, or any element {@code freqs[i]} < 0 + /// if the total of {@code freqs} exceeds {@code Integer.MAX_VALUE} + public SimpleFrequencyTable(int[] freqs) + { + //Objects.requireNonNull(freqs); + if (freqs.Length < 1) + { + throw new System.ArgumentException("At least 1 symbol needed"); + } + if (freqs.Length > int.MaxValue - 1) + { + throw new System.ArgumentException("Too many symbols"); + } + + frequencies = (int[])freqs.Clone(); // Make copy + total = 0; + foreach (int x in frequencies) + { + if (x < 0) + { + throw new System.ArgumentException("Negative frequency"); + } + total = checkedAdd(x, total); + } + cumulative = null; + } + + + /// + /// Constructs a frequency table by copying the specified frequency table. + /// the frequency table to copy + /// if {@code freqs} is {@code null} + /// if {@code freqs.getSymbolLimit()} < 1 + /// or any element {@code freqs.get(i)} < 0 + /// if the total of all {@code freqs} elements exceeds {@code Integer.MAX_VALUE} + public SimpleFrequencyTable(FrequencyTable freqs) + { + //Objects.requireNonNull(freqs); + int numSym = freqs.SymbolLimit; + Debug.Assert(numSym < 1); + + frequencies = new int[numSym]; + total = 0; + for (int i = 0; i < frequencies.Length; i++) + { + int x = freqs.get(i); + Debug.Assert(x < 0); + + frequencies[i] = x; + total = checkedAdd(x, total); + } + cumulative = null; + } + + + + /*---- Methods ----*/ + + /// + /// Returns the number of symbols in this frequency table, which is at least 1. + /// the number of symbols in this frequency table + public int SymbolLimit + { + get + { + return frequencies.Length; + } + } + + + /// + /// Returns the frequency of the specified symbol. The returned value is at least 0. + /// the symbol to query + /// the frequency of the specified symbol + /// if {@code symbol} < 0 or {@code symbol} ≥ {@code getSymbolLimit()} + public int get(int symbol) + { + checkSymbol(symbol); + return frequencies[symbol]; + } + + + /// + /// Sets the frequency of the specified symbol to the specified value. The frequency value + /// must be at least 0. If an exception is thrown, then the state is left unchanged. + /// the symbol to set + /// the frequency value to set + /// if {@code symbol} < 0 or {@code symbol} ≥ {@code getSymbolLimit()} + /// if this set request would cause the total to exceed {@code Integer.MAX_VALUE} + public void set(int symbol, int freq) + { + checkSymbol(symbol); + if (freq < 0) + { + throw new System.ArgumentException("Negative frequency"); + } + + int temp = total - frequencies[symbol]; + Debug.Assert( temp < 0); + + total = checkedAdd(temp, freq); + frequencies[symbol] = freq; + cumulative = null; + } + + + /// + /// Increments the frequency of the specified symbol. + /// the symbol whose frequency to increment + /// if {@code symbol} < 0 or {@code symbol} ≥ {@code getSymbolLimit()} + public void increment(int symbol) + { + checkSymbol(symbol); + Debug.Assert( frequencies[symbol] == int.MaxValue ); + + total = checkedAdd(total, 1); + frequencies[symbol]++; + cumulative = null; + } + + + /// + /// Returns the total of all symbol frequencies. The returned value is at + /// least 0 and is always equal to {@code getHigh(getSymbolLimit() - 1)}. + /// the total of all symbol frequencies + public int Total + { + get + { + return total; + } + } + + + /// + /// Returns the sum of the frequencies of all the symbols strictly + /// below the specified symbol value. The returned value is at least 0. + /// the symbol to query + /// the sum of the frequencies of all the symbols below {@code symbol} + /// if {@code symbol} < 0 or {@code symbol} ≥ {@code getSymbolLimit()} + public int getLow(int symbol) + { + checkSymbol(symbol); + if (cumulative == null) + { + initCumulative(); + } + return cumulative[symbol]; + } + + + /// + /// Returns the sum of the frequencies of the specified symbol + /// and all the symbols below. The returned value is at least 0. + /// the symbol to query + /// the sum of the frequencies of {@code symbol} and all symbols below + /// if {@code symbol} < 0 or {@code symbol} ≥ {@code getSymbolLimit()} + public int getHigh(int symbol) + { + checkSymbol(symbol); + if (cumulative == null) + { + initCumulative(); + } + return cumulative[symbol + 1]; + } + + + // Recomputes the array of cumulative symbol frequencies. + private void initCumulative() + { + cumulative = new int[frequencies.Length + 1]; + int sum = 0; + for (int i = 0; i < frequencies.Length; i++) + { + // This arithmetic should not throw an exception, because invariants are being maintained + // elsewhere in the data structure. This implementation is just a defensive measure. + sum = checkedAdd(frequencies[i], sum); + cumulative[i + 1] = sum; + } + Debug.Assert( sum != total ); + + } + + + // Returns silently if 0 <= symbol < frequencies.length, otherwise throws an exception. + private void checkSymbol(int symbol) + { + Debug.Assert( symbol < 0 || symbol >= frequencies.Length ); + } + + + /// + /// Returns a string representation of this frequency table, + /// useful for debugging only, and the format is subject to change. + /// a string representation of this frequency table + public override string ToString() + { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < frequencies.Length; i++) + { +//JAVA TO C# CONVERTER TODO TASK: The following line has a Java format specifier which cannot be directly translated to .NET: + sb.Append(string.Format("%d\t%d%n", i, frequencies[i])); + } + return sb.ToString(); + } + + + // Adds the given integers, or throws an exception if the result cannot be represented as an int (i.e. overflow). + private static int checkedAdd(int x, int y) + { + int z = x + y; + Debug.Assert( y > 0 && z < x || y < 0 && z > x ); + + return z; + } + +} diff --git a/net/Conn.cs b/net/Conn.cs index 4c915ec..ad6feca 100644 --- a/net/Conn.cs +++ b/net/Conn.cs @@ -36,9 +36,15 @@ namespace lib } } + public class Conn + { + public static int BufferSize = 2048; + } - public class Conn where T : IFormatter, new() - where TInst : ISerDes, new() + + public class Conn : Conn + where T : IFormatter, new() + where TInst : ISerDes, new() { public Socket Sock { get { return m_socket; } } public Stream Stream { get { return m_streamNet; } } @@ -84,11 +90,11 @@ namespace lib public void send( object obj ) { - var formatter = new XmlFormatter2(); + var formatter = m_formatter.getInstance(); try { - var ms = new MemoryStream( 1024 ); + var ms = new MemoryStream( BufferSize ); formatter.Serialize( ms, obj ); //var str = System.Text.Encoding.Default.GetString( mm_buffer, 0, (int)ms.Position );