Miko/tar1090
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move zstddec into own repository

Browse Source
This commit is contained in:
Matthias Wirth
2022-08-04 20:02:36 +02:00
parent 41e0f8d732
commit 0c5eedca39
7 changed files with 8 additions and 15454 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -517,6 +517,10 @@ alternative display style: &realHeat
<https://github.com/adsbxchange/wiki/wiki/tar1090-offline-map>
## Uses this library for decompressing zstd
<https://github.com/wiedehopf/zstddec-tar1090>
## NO WARRANTY - Excerpt from the License:
11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY

10
html/libs/zstddec-tar1090-0.0.5.js
View File

@@ -1,12 +1,10 @@
// Derived from https://github.com/donmccurdy/zstddec with modifications for compatibility by wiedehopf (matthias.wirth@gmail.com)
// LICENSE: JavaScript wrapper is provided under the MIT License
// LICENSE: WASM ZSTD decoder binary is provided under the BSD 3-Clause License, Copyright (c) 2016-2021, Yann Collet, Facebook, Inc. All rights reserved.
/*
* WASM ZSTD compiled by wiedehopf (matthias.wirth@gmail.com) 2022, for compatibility will return UINT32_MAX / 0xffffffff on error in findDecompressedSize, decompressed filesize limited to 4 GB
* See notes https://github.com/wiedehopf/tar1090/tree/master/zstddec/notes.md
*/
// Derived from https://github.com/donmccurdy/zstddec with modifications for compatibility by wiedehopf (matthias.wirth@gmail.com)
// See https://github.com/wiedehopf/zstddec-tar1090 for more info
// For browser compatibility, will return UINT32_MAX / 0xffffffff on error in findDecompressedSize, decompressed filesize limited to 4 GB
"use strict";

30
zstddec/LICENSE
View File

@@ -1,30 +0,0 @@
BSD License
For Zstandard software
Copyright (c) 2016-present, Facebook, Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name Facebook nor the names of its contributors may be used to
endorse or promote products derived from this software without specific
prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

78
zstddec/changes.diff
View File

@@ -1,78 +0,0 @@
--- /tmp/zstd-1.5.0/build/single_file_libs/zstddeclib.c 2022-07-19 08:40:31.371785257 +0200
+++ zstddec/zstddeclib.c 2022-07-19 08:52:36.644646467 +0200
@@ -16,6 +16,13 @@
* in the COPYING file in the root directory of this source tree).
* You may select, at your option, one of the above-listed licenses.
*/
+
+/*
+ * findDecompressedSize interface changed from unsigned long long to U32 for wasm / browser compatibility (matthias.wirth@gmail.com)
+ * further info on the issue: https://github.com/WebAssembly/WASI/issues/54
+ * This is not necessary for browsers after 2020, but there are still plenty of older devices around
+ */
+
/*
* Settings to bake for the standalone decompressor.
*
@@ -4937,7 +4944,7 @@
* note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to
* read each contained frame header. This is fast as most of the data is skipped,
* however it does mean that all frame data must be present and valid. */
-ZSTDLIB_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize);
+ZSTDLIB_API U32 ZSTD_findDecompressedSize(const void* src, size_t srcSize);
/*! ZSTD_decompressBound() :
* `src` should point to the start of a series of ZSTD encoded and/or skippable frames
@@ -11974,17 +11981,19 @@
* `srcSize` must be the exact length of some number of ZSTD compressed and/or
* skippable frames
* @return : decompressed size of the frames contained */
-unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize)
+U32 ZSTD_findDecompressedSize(const void* src, size_t srcSize)
{
unsigned long long totalDstSize = 0;
+ U32 _error = 0xffffffff;
+
while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) {
U32 const magicNumber = MEM_readLE32(src);
if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) {
size_t const skippableSize = readSkippableFrameSize(src, srcSize);
if (ZSTD_isError(skippableSize)) {
- return ZSTD_CONTENTSIZE_ERROR;
+ return _error;
}
assert(skippableSize <= srcSize);
@@ -11994,15 +12003,15 @@
}
{ unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize);
- if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret;
+ if (ret >= ZSTD_CONTENTSIZE_ERROR) return _error;
/* check for overflow */
- if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR;
+ if (totalDstSize + ret < totalDstSize) return _error;
totalDstSize += ret;
}
{ size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize);
if (ZSTD_isError(frameSrcSize)) {
- return ZSTD_CONTENTSIZE_ERROR;
+ return _error;
}
src = (const BYTE *)src + frameSrcSize;
@@ -12010,9 +12019,9 @@
}
} /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */
- if (srcSize) return ZSTD_CONTENTSIZE_ERROR;
+ if (totalDstSize >= UINT32_MAX || srcSize) return _error;
- return totalDstSize;
+ return (U32) totalDstSize;
}
/** ZSTD_getDecompressedSize() :

109
zstddec/notes.md
View File

@@ -1,109 +0,0 @@
## wasm zstd decoder for tar1090
Adaptation of the javascript shim from: https://github.com/donmccurdy/zstddec
See html/libs/zstddec-tar1090-VERSION.js for the js shim with compiled wasm, ready for usage.
The base64 converted wasm is placed into the C variable, you can't miss the long line :)
If you're interested in compiling the wasm yourself, see the instructions further down.
### API:
```
import { ZSTDDecoder } from 'zstddec';
const decoder = new ZSTDDecoder();
await decoder.init();
// requires and returns Uint8Array
const decompressedArray = decoder.decode( compressedArray, uncompressedSize );
```
### Example:
Include the js file in your html:
```html
<script src="zstddec-tar1090-0.0.4.js"></script>
```
In your js, use code similar to this to use zstddec-tar1090:
```js
let zstdDecode = null;
function webAssemblyFail(e) {
zstdDecode = null;
console.log(e);
console.error("Error loading zstddec, probable cause: webassembly not present or not working");
zstd = false;
// possibly display an error if your page requires zstddec
}
zstddec.decoder = new zstddec.ZSTDDecoder();
zstddec.promise = zstddec.decoder.init().catch(e => webAssemblyFail(e));
zstdDecode = zstddec.decoder.decode;
function arraybufferRequest() {
let xhrOverride = new XMLHttpRequest();
xhrOverride.responseType = 'arraybuffer';
return xhrOverride;
}
function startPage() {
let req = jQuery.ajax({
url: "http://127.0.0.1/test.zstd", method: 'GET',
xhr: arraybufferRequest,
timeout: 15000,
});
req.done(function(data) {
let arr = new Uint8Array(data);
lastRequestSize = arr.byteLength;
let res;
try {
res = zstdDecode( arr, 0 );
} catch (e) {
console.error(e);
return;
}
// process the decompressed data, for example, make it a string and parse as json:
let dstring = String.fromCharCode.apply(null, dresult);
let djson = JSON.parse(dstring);
});
}
zstddec.promise.then(function() {
startPage();
});
```
### Compiling the wasm from source:
#### obtaining C source:
```
wget https://github.com/facebook/zstd/releases/download/v1.5.0/zstd-1.5.0.tar.gz
tar xf zstd-1.5.0.tar.gz
cd zstd-1.5.0/build/single_file_libs
./combine.sh -r ../../lib -o zstddeclib.c zstddeclib-in.c
cp zstddeclib.c ../../..
cd ../../..
```
#### apply patch
```
patch -u zstddeclib.c -i changes.diff
```
* findDecompressedSize interface changed from unsigned long long to U32 for wasm / browser compatibility (uncompressed filesize limit 4294967294 bytes)
* the diff to the file created by combine.sh is also provided: changes.diff
* the modified file zstddeclib.c required for compilation is included in this directory
#### compilation:
```
docker pull emscripten/emsdk:3.1.15
docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) emscripten/emsdk:3.1.15 emcc zstddeclib.c --no-entry -O3 -s EXPORTED_FUNCTIONS="['_ZSTD_decompress', '_ZSTD_findDecompressedSize', '_ZSTD_isError', '_malloc', '_free']" -s ALLOW_MEMORY_GROWTH=1 -s TOTAL_STACK=64kb -s TOTAL_MEMORY=2Mb -o zstddec.wasm
```
TOTAL_MEMORY is the initial heap size which grows as necessary.
### convert to base64 for direct inclusion in js file:
```
base64 -w0 zstddec.wasm > zstddec.wasm.base64
```

79
zstddec/zstddec-tar1090-0.0.5.js
View File

File diff suppressed because one or more lines are too long

15152
zstddec/zstddeclib.c
View File

File diff suppressed because it is too large Load Diff