diff --git a/filesystems/Makefile b/filesystems/Makefile index 7225d1423458..c61eae0c5e36 100644 --- a/filesystems/Makefile +++ b/filesystems/Makefile @@ -1,146 +1,147 @@ COMMENT = File systems and related utilities SUBDIR += R-cran-fs SUBDIR += acfgfs SUBDIR += afuse SUBDIR += archivemount SUBDIR += automount SUBDIR += avfs SUBDIR += befuse SUBDIR += bindfs SUBDIR += cfs SUBDIR += chironfs SUBDIR += clamfs SUBDIR += cramfs SUBDIR += cryptofs SUBDIR += curlftpfs SUBDIR += darling-dmg SUBDIR += davfs2 SUBDIR += dsbmc SUBDIR += dsbmc-cli SUBDIR += dsbmd SUBDIR += e2fsprogs SUBDIR += e2fsprogs-core SUBDIR += e2fsprogs-libblkid SUBDIR += e2tools SUBDIR += encfs SUBDIR += exfat SUBDIR += exfat-utils SUBDIR += ext2 SUBDIR += fstyp SUBDIR += fswatch-mon SUBDIR += fusefs-libs SUBDIR += fusefs-libs3 SUBDIR += genromfs SUBDIR += gphotofs SUBDIR += gvfs SUBDIR += hammer2 SUBDIR += hfsexplorer SUBDIR += hfsfuse SUBDIR += hfsutils SUBDIR += httpdirfs SUBDIR += ifuse SUBDIR += jmtpfs SUBDIR += kio-fuse SUBDIR += libbde SUBDIR += libfsapfs SUBDIR += libfsext SUBDIR += libfsfat SUBDIR += libfshfs SUBDIR += libfsntfs SUBDIR += libfsxfs SUBDIR += libmodi SUBDIR += linux-c7-dosfstools SUBDIR += lizardfs SUBDIR += lkl SUBDIR += ltfs SUBDIR += mergerfs SUBDIR += mhddfs SUBDIR += moosefs2-cgi SUBDIR += moosefs2-cgiserv SUBDIR += moosefs2-chunkserver SUBDIR += moosefs2-cli SUBDIR += moosefs2-client SUBDIR += moosefs2-master SUBDIR += moosefs2-metalogger SUBDIR += moosefs2-netdump SUBDIR += moosefs3-cgi SUBDIR += moosefs3-cgiserv SUBDIR += moosefs3-chunkserver SUBDIR += moosefs3-cli SUBDIR += moosefs3-client SUBDIR += moosefs3-master SUBDIR += moosefs3-metalogger SUBDIR += moosefs3-netdump SUBDIR += mp3fs SUBDIR += mtools SUBDIR += mtpfs SUBDIR += nbt SUBDIR += ntfs SUBDIR += ntfs-compression SUBDIR += ntfs2btrfs SUBDIR += ods2 SUBDIR += openzfs SUBDIR += openzfs-kmod SUBDIR += p5-Filesys-Statvfs SUBDIR += p5-Fuse SUBDIR += pear-Horde_Vfs SUBDIR += pod SUBDIR += progsreiserfs SUBDIR += py-dfvfs SUBDIR += py-fs SUBDIR += py-fs2 SUBDIR += py-fsspec SUBDIR += py-fsspec-xrootd SUBDIR += py-fusepy SUBDIR += py-gcsfs + SUBDIR += py-kerchunk SUBDIR += py-libzfs SUBDIR += py-llfuse SUBDIR += py-prometheus-zfs SUBDIR += py-pyfakefs SUBDIR += py-s3fs SUBDIR += py-zfs-autobackup SUBDIR += rar2fs SUBDIR += rubygem-chef-winrm-fs SUBDIR += rubygem-winrm-fs SUBDIR += s3backer SUBDIR += s3fs SUBDIR += sandboxfs SUBDIR += scan_ffs SUBDIR += securefs SUBDIR += simple-mtpfs SUBDIR += smbnetfs SUBDIR += squashfs-tools SUBDIR += squashfs-tools-ng SUBDIR += squashfuse SUBDIR += sshfs SUBDIR += tclvfs SUBDIR += ufs SUBDIR += ufs_copy SUBDIR += unionfs SUBDIR += unreliablefs SUBDIR += vzvol SUBDIR += webdavfs SUBDIR += wimlib SUBDIR += xfsm SUBDIR += xfsprogs SUBDIR += xfuse SUBDIR += zap SUBDIR += zfs-periodic SUBDIR += zfs-replicate SUBDIR += zfs-snap-diff SUBDIR += zfs-snapshot-clean SUBDIR += zfs-snapshot-mgmt SUBDIR += zfs-stats SUBDIR += zfs-stats-lite SUBDIR += zfsnap SUBDIR += zfsnap2 SUBDIR += zfstools SUBDIR += zip SUBDIR += zisofs-tools SUBDIR += zrepl SUBDIR += zrepl-dsh2dsh SUBDIR += ztop SUBDIR += zxfer .include diff --git a/filesystems/py-kerchunk/Makefile b/filesystems/py-kerchunk/Makefile new file mode 100644 index 000000000000..4fef8b7643e6 --- /dev/null +++ b/filesystems/py-kerchunk/Makefile @@ -0,0 +1,29 @@ +PORTNAME= kerchunk +PORTVERSION= 0.2.7 +CATEGORIES= filesystems python +MASTER_SITES= PYPI +PKGNAMEPREFIX= ${PYTHON_PKGNAMEPREFIX} + +MAINTAINER= sunpoet@FreeBSD.org +COMMENT= Functions to make reference descriptions for ReferenceFileSystem +WWW= https://fsspec.github.io/kerchunk/ \ + https://github.com/fsspec/kerchunk + +LICENSE= MIT +LICENSE_FILE= ${WRKSRC}/LICENSE + +BUILD_DEPENDS= ${PYTHON_PKGNAMEPREFIX}setuptools>=42:devel/py-setuptools@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}setuptools-scm>=7:devel/py-setuptools-scm@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}wheel>=0:devel/py-wheel@${PY_FLAVOR} +RUN_DEPENDS= ${PYTHON_PKGNAMEPREFIX}fsspec>=0:filesystems/py-fsspec@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}numcodecs>=0:misc/py-numcodecs@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}numpy>=0,1:math/py-numpy@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}ujson>=0:devel/py-ujson@${PY_FLAVOR} \ + ${PYTHON_PKGNAMEPREFIX}zarr>=0.1<3,1:devel/py-zarr@${PY_FLAVOR} + +USES= python +USE_PYTHON= autoplist concurrent pep517 + +NO_ARCH= yes + +.include diff --git a/filesystems/py-kerchunk/distinfo b/filesystems/py-kerchunk/distinfo new file mode 100644 index 000000000000..45262a4ccc15 --- /dev/null +++ b/filesystems/py-kerchunk/distinfo @@ -0,0 +1,3 @@ +TIMESTAMP = 1748107898 +SHA256 (kerchunk-0.2.7.tar.gz) = 0425aa0fbf56f898053ee4c4dd40b35cea12d2fc986e036086e99a4ad16bd4e6 +SIZE (kerchunk-0.2.7.tar.gz) = 709052 diff --git a/filesystems/py-kerchunk/pkg-descr b/filesystems/py-kerchunk/pkg-descr new file mode 100644 index 000000000000..a351e30943fe --- /dev/null +++ b/filesystems/py-kerchunk/pkg-descr @@ -0,0 +1,28 @@ +Kerchunk is a library that provides a unified way to represent a variety of +chunked, compressed data formats (e.g. NetCDF, HDF5, GRIB), allowing efficient +access to the data from traditional file systems or cloud object storage. It +also provides a flexible way to create virtual datasets from multiple files. It +does this by extracting the byte ranges, compression information and other +information about the data and storing this metadata in a new, separate object. +This means that you can create a virtual aggregate dataset over potentially many +source files, for efficient, parallel and cloud-friendly in-situ access without +having to copy or translate the originals. It is a gateway to in-the-cloud +massive data processing while the data providers still insist on using legacy +formats for archival storage. + +We provide the following things: +- completely serverless architecture +- metadata consolidation, so you can understand a many-file dataset (metadata + plus physical storage) in a single read +- read from all of the storage backends supported by fsspec, including object + storage (s3, gcs, abfs, alibaba), http, cloud user storage (dropbox, gdrive) + and network protocols (ftp, ssh, hdfs, smb...) +- loading of various file types (currently netcdf4/HDF, grib2, tiff, fits, + zarr), potentially heterogeneous within a single dataset, without a need to go + via the specific driver (e.g., no need for h5py) +- asynchronous concurrent fetch of many data chunks in one go, amortizing the + cost of latency +- parallel access with a library like zarr without any locks +- logical datasets viewing many (>~millions) data files, and direct + access/subselection to them via coordinate indexing across an arbitrary number + of dimensions