Compare commits

...

117 Commits

Author SHA1 Message Date
dependabot[bot]
9259344452 Bump golang.org/x/net from 0.36.0 to 0.38.0
Bumps [golang.org/x/net](https://github.com/golang/net) from 0.36.0 to 0.38.0.
- [Commits](https://github.com/golang/net/compare/v0.36.0...v0.38.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-version: 0.38.0
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-05-28 15:18:33 +01:00
Han Verstraete (OpenFaaS Ltd)
83e804513a Fix faasd CE install script for RHEL-based systems
This change removes the check-update command. This command is not
required and caused the script to exit early if packages are not up to
date.

In addition DNF is now used to install packages. DNF is the successor of
YUM on the latest RHEL-based system.

Signed-off-by: Han Verstraete (OpenFaaS Ltd) <han@openfaas.com>
2025-05-20 17:21:08 +01:00
Han Verstraete (OpenFaaS Ltd)
f409e01aa6 Don't install weak dependencies on RHEL-based systems
Signed-off-by: Han Verstraete (OpenFaaS Ltd) <han@openfaas.com>
2025-05-20 17:21:08 +01:00
Han Verstraete (OpenFaaS Ltd)
e98c949019 Add allowerasing flag to dnf install
Fixes issues with conflicting packages that may occur on some systems.

Signed-off-by: Han Verstraete (OpenFaaS Ltd) <han@openfaas.com>
2025-05-20 13:53:26 +01:00
Han Verstraete (OpenFaaS Ltd)
6f0ac37e3c Add bridge-utils to RHEL packages
Signed-off-by: Han Verstraete (OpenFaaS Ltd) <han@openfaas.com>
2025-05-20 13:29:27 +01:00
Han Verstraete (OpenFaaS Ltd)
0defc22a3f Rename missed has_yum reference to has_dnf
Signed-off-by: Han Verstraete (OpenFaaS Ltd) <han@openfaas.com>
2025-05-20 13:29:27 +01:00
Han Verstraete (OpenFaaS Ltd)
c14d92c5c2 Use full faasd binary path in instructions on RHEL
On RHEL systems the installation path of faasd, /usr/local/bin, is not
in the sudo PATH by default. Use the full path to the faasd binary in
any printed instruction to work around this.

Signed-off-by: Han Verstraete (OpenFaaS Ltd) <han@openfaas.com>
2025-05-14 12:43:33 +01:00
Han Verstraete (OpenFaaS Ltd)
934d326d82 Fix OpenFaaS Edge install script for RHEL-based systems
This change removes the check-update command. This command is not
required and caused the script to exit early if packages are not up to
date.

In addition DNF is now used to install packages. DNF is the successor of
YUM on the latest RHEL-based systems.

Tested on Rocky Linux 8 and 9.

Signed-off-by: Han Verstraete (OpenFaaS Ltd) <han@openfaas.com>
2025-05-14 12:43:33 +01:00
Alex Ellis (OpenFaaS Ltd)
0ea1bd5ffc Invert order of instructions for edge
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-04-24 16:09:18 +01:00
Alex Ellis (OpenFaaS Ltd)
7f798267b0 Add note on docs
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-04-24 16:07:57 +01:00
Alex Ellis (OpenFaaS Ltd)
a2254ca1ff Security fix - containerd to 1.7.27
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-03-18 10:25:34 +00:00
Alex Ellis (OpenFaaS Ltd)
087a299f4c Add service logs command to faasd ce from faasd-pro
A more limited version is added, for continuity.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-03-18 10:23:22 +00:00
dependabot[bot]
6dcdab832d Bump golang.org/x/net from 0.24.0 to 0.36.0
Bumps [golang.org/x/net](https://github.com/golang/net) from 0.24.0 to 0.36.0.
- [Commits](https://github.com/golang/net/compare/v0.24.0...v0.36.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-03-13 08:36:45 +00:00
Alex Ellis (OpenFaaS Ltd)
57163e27cf Revised README
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-02-21 10:38:19 +00:00
Alex Ellis (OpenFaaS Ltd)
cced8eb3df OpenFaaS CE EULA is for personal use and limited commercial trial
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-02-21 09:32:31 +00:00
Alex Ellis (OpenFaaS Ltd)
0d4bfa938c Add script to patch DigitalOcean for journal
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-02-19 11:26:02 +00:00
Alex Ellis (OpenFaaS Ltd)
120a34cfef Update post-installation message on container downloads
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-02-19 11:16:06 +00:00
Alex Ellis (OpenFaaS Ltd)
ae4d0a97f8 Add directive back into install-edge.sh
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-02-19 11:12:03 +00:00
Alex Ellis (OpenFaaS Ltd)
ef68994a5c Install script - suppress interactive messages
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-02-19 11:09:36 +00:00
Alex Ellis (OpenFaaS Ltd)
43e51c51bb Show instructions for manual license keys
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-02-19 11:08:19 +00:00
Alex Ellis (OpenFaaS Ltd)
695d80076c Suppress question about autosave for iptables
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-02-19 11:03:00 +00:00
Alex Ellis (OpenFaaS Ltd)
51f3f87ba1 Add iptables-persistent for openfaas-edge on Debian-like OSes
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-02-19 11:01:38 +00:00
Alex Ellis (OpenFaaS Ltd)
e2758be25e Fix file permission
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-02-18 21:17:07 +00:00
Alex Ellis (OpenFaaS Ltd)
7ad5c17e7c Clarify usage in a Small Business Environment
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-02-17 17:25:17 +00:00
Alex Ellis
983dacb15c
Create config.yml
Signed-off-by: Alex Ellis <alexellis2@gmail.com>
2025-01-30 10:50:30 +00:00
Alex Ellis
96abaaf102
Create issue.md
Signed-off-by: Alex Ellis <alexellis2@gmail.com>
2025-01-30 10:37:32 +00:00
Alex Ellis (OpenFaaS Ltd)
d7e0bebe25 Fix for proxy exiting early
When a proxied core service was accessed before it was ready
to accept a connection, due to a start-up, restart, etc of
a core service, then the proxy exited instead of continuing
to accept new connections.

This meant having to restart faasd and hope the race condition
worked itself out, or that no incoming requests were made.

Tested with Grafana, which seemed to manifest the issue
the most.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-01-30 10:00:14 +00:00
Alex Ellis (OpenFaaS Ltd)
ef689d7b62 Fix for update workflow
When terminating a function and replacing it during an update,
there was often an error about task precondition not met
which meant having to try and or wait or being left in an
inconsistent state.

The new flow makes sure "Wait" is called in either code path
and allows for a custom gap between the SIGTERM and SIGKILL
through the grace_period env var - set as a Go duration.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-01-21 17:20:30 +00:00
Alex Ellis (OpenFaaS Ltd)
854ec5836d Remove unused context
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-01-21 16:03:17 +00:00
Alex Ellis (OpenFaaS Ltd)
4ab5f60b9d Reduce logging from log streaming command
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-01-21 12:58:00 +00:00
Alex Ellis (OpenFaaS Ltd)
a8b61f2086 Upgrade various core components
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2025-01-21 12:54:54 +00:00
Alex Ellis (OpenFaaS Ltd)
68335e2016 Update images for Go 1.23
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-12-13 08:22:44 +00:00
Alex Ellis (OpenFaaS Ltd)
404af1c4d9 Fix #363 with format for max memory
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-11-22 08:47:07 +00:00
Alex Ellis (OpenFaaS Ltd)
055e57ec5f Clarify EULA for CE components
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-11-21 16:39:57 +00:00
Alex Ellis (OpenFaaS Ltd)
6bb1222ebb Switch to newest gateway version and Prometheus v3
Fixes: #371

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-11-21 16:37:56 +00:00
Alex Ellis (OpenFaaS Ltd)
599ae5415f Force BINLOCATION to arkade install
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-25 11:18:56 +01:00
Alex Ellis (OpenFaaS Ltd)
0d74cac072 Kill existing services before installing faasd edge
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-25 10:22:45 +01:00
Alex Ellis (OpenFaaS Ltd)
c2b802cbf9 Stop edge installer on error
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-25 10:15:32 +01:00
Alex Ellis (OpenFaaS Ltd)
bd0e1d7718 Update support for Oracle Linux for edge installer
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-25 10:15:00 +01:00
Alex Ellis (OpenFaaS Ltd)
bfc87ff432 Ensure which is available for ym
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-25 09:48:48 +01:00
Alex Ellis (OpenFaaS Ltd)
032716e3e9 Update OpenFaaS Edge metrics
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-24 15:34:26 +01:00
Alex Ellis (OpenFaaS Ltd)
c813b0810b Notes in install-edge script
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-23 14:51:49 +01:00
Alex Ellis (OpenFaaS Ltd)
fb36d2e5aa Update notes on OpenFaaS edge
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-23 14:17:49 +01:00
Alex Ellis (OpenFaaS Ltd)
038eb91191 Update script URL for edge
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-23 14:11:45 +01:00
Alex Ellis (OpenFaaS Ltd)
5576382d96 Install OS package via faasd-pro script
These packages need to be sourced by other means when
installing into an airgap, and you can set SKIP_OS=1 to avoid
using the Internet.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-23 09:34:41 +01:00
Alex Ellis (OpenFaaS Ltd)
7ca2621c98 Add notes about other OS packages for faasd-pro
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-20 09:42:36 +01:00
Alex Ellis (OpenFaaS Ltd)
a154fd1bc0 Add Pro to README
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-18 16:36:45 +01:00
Alex Ellis (OpenFaaS Ltd)
6b1e49a2a5 Ensure scaling up from zero works in faasd CE
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-18 16:33:20 +01:00
Alex Ellis (OpenFaaS Ltd)
5344a32472 Ensure scaling up from zero works in faasd CE
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-18 15:45:13 +01:00
Alex Ellis (OpenFaaS Ltd)
e59e3f0cb6 Add pro installer for sponsors
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-18 15:42:38 +01:00
Alex Ellis (OpenFaaS Ltd)
2adc1350d4 Fix import
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-17 15:34:45 +01:00
Alex Ellis (OpenFaaS Ltd)
5b633cc017 Remove newlines from log lines, which are not required
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-17 11:18:07 +01:00
Alex Ellis (OpenFaaS Ltd)
1c1bfa6759 Upgrades to containerd, fix deprecations upstream
* Fixes upstream deprecations for containerd
* Fixes deprecations in ioutil package usage
* Break out separate files for function handlers
* Upgrades containerd to 1.7.22
* Fixes default namespace functionality
* Pre-deploy checks as per license agreement
* Removes extra log messages for payload in HTTP handlers, you
can enable FAAS_DEBUG=1 in the CLI instead to see this from
the client's perspective
* Add additional Google DNS server 8.8.4.4 for functions

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-17 11:13:53 +01:00
Alex Ellis (OpenFaaS Ltd)
93f41ca35d Clarify coverage of OpenFaaS CE EULA
faasd has always been covered by the OpenFaaS CE EULA, the
presence of the document in the faas repo does not mean it
only applies to a single repository.

faasd is OpenFaaS, they are not different products.

OpenFaaS CE requires a commercial license for use as has been
explained on the website and in the docs.

https://www.openfaas.com/pricing
https://docs.openfaas.com/openfaas-pro/introduction/#comparison

This EULA Addendum permits explit personal usage and small
business environments to commercial usage of faasd only, with
OpenFaaS CE, but does not cover OpenFaaS CE deployed to
Kubernetes or K3s, or OpenShift.

For questions or comments, reach out to contact@openfaas.com

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-09-17 08:37:39 +01:00
dependabot[bot]
0172c996b8 Bump github.com/opencontainers/runc from 1.1.12 to 1.1.14
Bumps [github.com/opencontainers/runc](https://github.com/opencontainers/runc) from 1.1.12 to 1.1.14.
- [Release notes](https://github.com/opencontainers/runc/releases)
- [Changelog](https://github.com/opencontainers/runc/blob/main/CHANGELOG.md)
- [Commits](https://github.com/opencontainers/runc/compare/v1.1.12...v1.1.14)

---
updated-dependencies:
- dependency-name: github.com/opencontainers/runc
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-09-09 15:31:40 +01:00
Alex Ellis (OpenFaaS Ltd)
4162db43ff Fix check for arch in verify_system
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-06-12 10:54:18 +01:00
Alex Ellis (OpenFaaS Ltd)
e4848cd829 Remove armhf/armv7 support from faasd
See notes in: #364

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-06-12 10:19:41 +01:00
Alex Ellis (OpenFaaS Ltd)
7dbaeef3d8 Return error from proxy
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-06-12 10:02:05 +01:00
Alex Ellis (OpenFaaS Ltd)
887c18befa Close connection if unable to dial upstream
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-06-12 10:02:05 +01:00
dependabot[bot]
f6167e72a9 Bump golang.org/x/net from 0.22.0 to 0.23.0
Bumps [golang.org/x/net](https://github.com/golang/net) from 0.22.0 to 0.23.0.
- [Commits](https://github.com/golang/net/compare/v0.22.0...v0.23.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-04-23 10:35:50 +01:00
Alex Ellis (OpenFaaS Ltd)
c12505a63f Update intro, and build with Go 1.22
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-03-27 09:09:12 +00:00
Alex Ellis (OpenFaaS Ltd)
ffac4063b6 Updates for go.mod
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-03-20 09:41:37 +01:00
wangqiang
dd31784824 Fix snapshot key error
When obtaining the container information fails, the key for deleting the snapshot is incorrect.

Signed-off-by: wangqiang <shivaqiang@qq.com>
2024-03-20 09:40:17 +01:00
dependabot[bot]
70b0929cf2 Bump google.golang.org/protobuf from 1.31.0 to 1.33.0
Bumps google.golang.org/protobuf from 1.31.0 to 1.33.0.

---
updated-dependencies:
- dependency-name: google.golang.org/protobuf
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2024-03-14 14:50:22 +00:00
Alex Ellis (OpenFaaS Ltd)
c61efe06f4 Update runc
Closes: #355

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-02-01 10:03:35 +00:00
Alex Ellis (OpenFaaS Ltd)
b05844acea Remove line
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-01-12 11:04:13 +00:00
Alex Ellis (OpenFaaS Ltd)
fd20e69ee1 Update gateway version
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-01-11 18:22:38 +00:00
Alex Ellis (OpenFaaS Ltd)
4a3fa684e2 Updates for text streaming functions
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2024-01-11 18:13:34 +00:00
Alex Ellis (OpenFaaS Ltd)
f17a25f3e8 Update go.mod and Prometheus version
Updates various internal dependencies in go.mod, and Prometheus
within docker-compose.yaml

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-12-11 16:08:16 +00:00
dependabot[bot]
7ef56d8dae Bump google.golang.org/grpc from 1.53.0 to 1.56.3
Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.53.0 to 1.56.3.
- [Release notes](https://github.com/grpc/grpc-go/releases)
- [Commits](https://github.com/grpc/grpc-go/compare/v1.53.0...v1.56.3)

---
updated-dependencies:
- dependency-name: google.golang.org/grpc
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2023-12-01 09:37:21 +00:00
Nitishkumar Singh
1cb5493f72 mark namespace with label openfaas=1 valid
Signed-off-by: Nitishkumar Singh <nitishkumarsingh71@gmail.com>
2023-11-29 16:40:38 +00:00
Alex Ellis (OpenFaaS Ltd)
d85332be13 Update queue-worker, NATS, Prometheus and gateway
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-11-20 08:10:29 +00:00
Alex Ellis (OpenFaaS Ltd)
1412faffd2 Add Makefile target for updating images
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-10-23 11:04:48 +01:00
Alex Ellis (OpenFaaS Ltd)
2685c1db06 Upgrade to go-execute/v2
Upgrades to go-execute/v2 and updates various other
dependencies.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-10-23 11:02:37 +01:00
Nitishkumar Singh
078043b168 disable printing invocation timing in stderr
Signed-off-by: Nitishkumar Singh <nitishkumarsingh71@gmail.com>
2023-10-23 10:58:02 +01:00
dependabot[bot]
5356fca4c5 Bump golang.org/x/net from 0.10.0 to 0.17.0
Bumps [golang.org/x/net](https://github.com/golang/net) from 0.10.0 to 0.17.0.
- [Commits](https://github.com/golang/net/compare/v0.10.0...v0.17.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2023-10-20 13:52:08 +01:00
Alex Ellis (OpenFaaS Ltd)
99ccd75b62 Update Prometheus and CE gateway
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-10-02 13:56:44 +01:00
Peter Mills
aab5363e65 Implement read only flag
Signed-off-by: Peter Mills <ptjm8422@gmail.com>
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-09-27 10:34:00 +01:00
dependabot[bot]
ba601bfc67 Bump github.com/cyphar/filepath-securejoin from 0.2.3 to 0.2.4
Bumps [github.com/cyphar/filepath-securejoin](https://github.com/cyphar/filepath-securejoin) from 0.2.3 to 0.2.4.
- [Release notes](https://github.com/cyphar/filepath-securejoin/releases)
- [Commits](https://github.com/cyphar/filepath-securejoin/compare/v0.2.3...v0.2.4)

---
updated-dependencies:
- dependency-name: github.com/cyphar/filepath-securejoin
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2023-09-13 15:39:23 +01:00
Alex Ellis (OpenFaaS Ltd)
53670e2854 Update to latest fresh images for faasd
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-07-27 09:26:25 +01:00
Alex Ellis (OpenFaaS Ltd)
57baf34f5a Add support for namespace mutations
Adds support for: Get, Create, Update, Delete

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-07-07 13:45:36 +01:00
Alex Ellis (OpenFaaS Ltd)
c83b649301 Update to latest faas-provider
The scale and delete endpoints no-longer accept namespace
via the query string, but through the body.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-07-07 09:48:27 +01:00
Alex Ellis (OpenFaaS Ltd)
f394b4a2f1 Update Prometheus, NATS and GW CE images
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-07-06 11:05:06 +01:00
Alex Ellis (OpenFaaS Ltd)
d0219d6697 Build with Go 1.20 and update misc dependencies
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-07-06 11:04:06 +01:00
dependabot[bot]
7d073bd64b Bump github.com/docker/docker
Bumps [github.com/docker/docker](https://github.com/docker/docker) from 23.0.1+incompatible to 23.0.3+incompatible.
- [Release notes](https://github.com/docker/docker/releases)
- [Commits](https://github.com/docker/docker/compare/v23.0.1...v23.0.3)

---
updated-dependencies:
- dependency-name: github.com/docker/docker
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2023-05-23 11:20:58 +01:00
dependabot[bot]
ebec399817 Bump github.com/docker/distribution
Bumps [github.com/docker/distribution](https://github.com/docker/distribution) from 2.8.1+incompatible to 2.8.2+incompatible.
- [Release notes](https://github.com/docker/distribution/releases)
- [Commits](https://github.com/docker/distribution/compare/v2.8.1...v2.8.2)

---
updated-dependencies:
- dependency-name: github.com/docker/distribution
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
2023-05-23 11:05:52 +01:00
dependabot[bot]
553054ddf5 Bump github.com/opencontainers/runc from 1.1.4 to 1.1.5
Bumps [github.com/opencontainers/runc](https://github.com/opencontainers/runc) from 1.1.4 to 1.1.5.
- [Release notes](https://github.com/opencontainers/runc/releases)
- [Changelog](https://github.com/opencontainers/runc/blob/v1.1.5/CHANGELOG.md)
- [Commits](https://github.com/opencontainers/runc/compare/v1.1.4...v1.1.5)

---
updated-dependencies:
- dependency-name: github.com/opencontainers/runc
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2023-04-11 09:43:06 +01:00
Gianpaolo Macario
86d2797873 MULTIPASS.md: Fix typo
Remove extra "P" in "contents"

Signed-off-by: Gianpaolo Macario <gmacario@gmail.com>
2023-04-03 15:17:42 +01:00
Alex Ellis
0d3dc50f61
Update ISSUE_TEMPLATE.md
Signed-off-by: Alex Ellis <alexellis2@gmail.com>
2023-03-28 19:18:04 +01:00
Alex Ellis (OpenFaaS Ltd)
b1ea842fb1 Update flag for multipass
--mem is now deprecated in favour of --memory

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-03-21 17:01:48 +00:00
Alex Ellis (OpenFaaS Ltd)
e1d90bba60 Update Prometheus and OpenFaaS CE gateway versions
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-03-20 08:40:57 +00:00
Alex Ellis (OpenFaaS Ltd)
6c8214b27e Remove single item matrix from workflow files
Not required, or useful.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-03-20 08:40:57 +00:00
Alex Ellis (OpenFaaS Ltd)
990f1a4df6 Update to Go 1.19
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-03-20 08:40:57 +00:00
Alex Ellis (OpenFaaS Ltd)
c41c2cd9fc Migrate to containerd v1.7.0 and update dependencies
* Updates containerd to v1.7.0 and new binary for 32-bit
Arm OSes.
* Updates Go dependencies - openfaas and external

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-03-20 08:40:57 +00:00
Alex Ellis (OpenFaaS Ltd)
9efd019e86 Update vision
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-03-06 11:03:39 +00:00
Alex Ellis (OpenFaaS Ltd)
d09ab85bda Update notes in README
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-03-06 11:03:39 +00:00
dependabot[bot]
c5c0fa05a2 Bump golang.org/x/net from 0.0.0-20220722155237-a158d28d115b to 0.7.0
Bumps [golang.org/x/net](https://github.com/golang/net) from 0.0.0-20220722155237-a158d28d115b to 0.7.0.
- [Release notes](https://github.com/golang/net/releases)
- [Commits](https://github.com/golang/net/commits/v0.7.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>
2023-02-27 09:27:38 +00:00
Nitishkumar Singh
aaf1811052 Removed auth plugin
Signed-off-by: Nitishkumar Singh <nitishkumarsingh71@gmail.com>
2023-02-15 17:33:45 +00:00
Alex Ellis (OpenFaaS Ltd)
4d6b6dfdc5 Update images for NATS / GW / queue-worker
Routine updates to the containers that are used for faasd CE

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-01-17 22:08:44 +00:00
Alex Ellis (OpenFaaS Ltd)
b844a72067 Bump images for nats, Prometheus and OpenFaaS CE
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2023-01-03 15:17:47 +00:00
Alex Ellis
8227285faa
Update ISSUE_TEMPLATE.md
Signed-off-by: Alex Ellis <alexellis2@gmail.com>
2022-10-24 11:48:51 +01:00
Nitishkumar Singh
87773fd167 Verify docker-compose images
Signed-off-by: Nitishkumar Singh <nitishkumarsingh71@gmail.com>

Match all yaml files

Signed-off-by: Nitishkumar Singh <nitishkumarsingh71@gmail.com>

feedback changes

Signed-off-by: Nitishkumar Singh <nitishkumarsingh71@gmail.com>
2022-10-23 08:18:46 +01:00
Alex Ellis (OpenFaaS Ltd)
ecf82ec37b Add a second link to the ROADMAP
Whilst the ROADMAP was already linked, one user was confused
about the design decisions of faasd, so this adds a second
link to make it more likely he would click on it in the future.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2022-10-04 16:21:10 +01:00
Alex Ellis
9e8c680f3f
Update README.md
Signed-off-by: Alex Ellis <alexellis2@gmail.com>
2022-10-04 08:56:19 +01:00
Alex Ellis (OpenFaaS Ltd)
85c1082fac Add MaxIdleConns and MaxIdleConnsPerHost
This gives a default value, for belt and braces, for people
who are supplying a lot of load.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2022-10-03 18:22:56 +01:00
Alex Ellis (OpenFaaS Ltd)
282b05802c Suppress error for faas-cli list during deletion
Fixes: #306

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2022-10-03 17:28:24 +01:00
Alex Ellis (OpenFaaS Ltd)
7c118225b2 Update apimachinery
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2022-10-03 16:46:59 +01:00
Alex Ellis (OpenFaaS Ltd)
95792f8d58 Update dependencies
* Updates netlink/netns
* Updates x/sys, arkade and apimachinery

Build passes, minor updates.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2022-10-03 16:43:35 +01:00
Alex Ellis (OpenFaaS Ltd)
60b724f014 Update gateway version
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2022-10-03 16:40:17 +01:00
Alex Ellis
e0db59d8a1
Update README.md
Signed-off-by: Alex Ellis <alexellis2@gmail.com>
2022-08-18 16:20:22 +01:00
Alex Ellis (OpenFaaS Ltd)
13304fa0b2 Fix for user issue with Caddy
Caddy cannot be installed to /usr/local/bin due to the
service file expecting it to be in /usr/bin

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2022-08-18 16:13:54 +01:00
Han Verstraete
a65b989b15 Fix install script
Remove v prefix from CONTAINERD_VER variable.

Signed-off-by: Han Verstraete <han@openfaas.com>
2022-08-18 12:43:43 +01:00
Alex Ellis (OpenFaaS Ltd)
6b6ff71c29 Update to containerd 1.6.8
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2022-08-18 12:15:33 +01:00
Alex Ellis (OpenFaaS Ltd)
bb5b212663 Fix installation script for containerd on armhf
Fixes #295

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2022-08-18 11:19:13 +01:00
Alex Ellis (OpenFaaS Ltd)
9564e64980 Fix issue with provider metrics
https://github.com/openfaas/faas-provider/pull/66

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
2022-08-17 20:33:58 +01:00
dependabot[bot]
6dbc33d045 Bump github.com/containerd/containerd from 1.6.4 to 1.6.6
Bumps [github.com/containerd/containerd](https://github.com/containerd/containerd) from 1.6.4 to 1.6.6.
- [Release notes](https://github.com/containerd/containerd/releases)
- [Changelog](https://github.com/containerd/containerd/blob/main/RELEASES.md)
- [Commits](https://github.com/containerd/containerd/compare/v1.6.4...v1.6.6)

---
updated-dependencies:
- dependency-name: github.com/containerd/containerd
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
2022-08-16 14:41:51 +01:00
Nitishkumar Singh
5cedf28929 Dev.md updated to version 0.16.2
Signed-off-by: Nitishkumar Singh <nitishkumarsingh71@gmail.com>

updated faasd version in terraform script

Signed-off-by: Nitishkumar Singh <nitishkumarsingh71@gmail.com>
2022-07-19 13:08:50 +01:00
2011 changed files with 204683 additions and 115269 deletions

View File

@ -8,6 +8,10 @@ Before you ask for help or support, make sure that you've [consulted the manual
<!-- How is this affecting you? What task are you trying to accomplish? -->
## Why do you need this?
<!-- Attempts to mask or hide this may result in the issue being closed -->
## Who is this for?
What company is this for? Are you listed in the [ADOPTERS.md](https://github.com/openfaas/faas/blob/master/ADOPTERS.md) file?
<!--- Provide a general summary of the issue in the Title above -->
## Expected Behaviour
@ -20,13 +24,6 @@ Before you ask for help or support, make sure that you've [consulted the manual
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
## Are you a GitHub Sponsor (Yes/No?)
<!--- Given this request for help, how are you supporting the project? -->
Check at: https://github.com/sponsors/openfaas
- [ ] Yes
- [ ] No
## List All Possible Solutions and Workarounds
<!--- Suggest a fix/reason for the bug, or ideas how to implement -->
<!--- the addition or change -->

1
.github/ISSUE_TEMPLATE/config.yml vendored Normal file
View File

@ -0,0 +1 @@
blank_issues_enabled: false

69
.github/ISSUE_TEMPLATE/issue.md vendored Normal file
View File

@ -0,0 +1,69 @@
---
name: Report an issue
about: Create a report to help us improve
title: ''
labels: ''
assignees: ''
---
## Due diligence
<!-- Due dilligence -->
## My actions before raising this issue
Before you ask for help or support, make sure that you've [consulted the manual for faasd](https://openfaas.gumroad.com/l/serverless-for-everyone-else). We can't answer questions that are already covered by the manual.
<!-- How is this affecting you? What task are you trying to accomplish? -->
## Why do you need this?
<!-- Attempts to mask or hide this may result in the issue being closed -->
## Who is this for?
What company is this for? Are you listed in the [ADOPTERS.md](https://github.com/openfaas/faas/blob/master/ADOPTERS.md) file?
<!--- Provide a general summary of the issue in the Title above -->
## Expected Behaviour
<!--- If you're describing a bug, tell us what should happen -->
<!--- If you're suggesting a change/improvement, tell us how it should work -->
## Current Behaviour
<!--- If describing a bug, tell us what happens instead of the expected behavior -->
<!--- If suggesting a change/improvement, explain the difference from current behavior -->
## List All Possible Solutions and Workarounds
<!--- Suggest a fix/reason for the bug, or ideas how to implement -->
<!--- the addition or change -->
<!--- Is there a workaround which could avoid making changes? -->
## Which Solution Do You Recommend?
<!--- Pick your preferred solution, if you were to implement and maintain this change -->
## Steps to Reproduce (for bugs)
<!--- Provide a link to a live example, or an unambiguous set of steps to -->
<!--- reproduce this bug. Include code to reproduce, if relevant -->
1.
2.
3.
4.
## Your Environment
* OS and architecture:
* Versions:
```sh
go version
containerd -version
uname -a
cat /etc/os-release
faasd version
```

View File

@ -10,19 +10,15 @@ jobs:
build:
env:
GO111MODULE: off
strategy:
matrix:
go-version: [1.18.x]
os: [ubuntu-latest]
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
with:
fetch-depth: 1
- name: Install Go
uses: actions/setup-go@v2
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: 1.22.x
- name: test
run: make test

View File

@ -7,23 +7,19 @@ on:
jobs:
publish:
strategy:
matrix:
go-version: [ 1.18.x ]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
with:
fetch-depth: 1
- name: Install Go
uses: actions/setup-go@v2
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: 1.22.x
- name: Make publish
run: make publish
- name: Upload release binaries
uses: alexellis/upload-assets@0.2.2
uses: alexellis/upload-assets@0.4.1
env:
GITHUB_TOKEN: ${{ github.token }}
with:

18
.github/workflows/verify-images.yaml vendored Normal file
View File

@ -0,0 +1,18 @@
name: Verify Docker Compose Images
on:
push:
paths:
- '**.yaml'
jobs:
verifyImages:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- uses: alexellis/setup-arkade@v3
- name: Verify chart images
id: verify_images
run: |
VERBOSE=true make verify-compose

22
EULA.md Normal file
View File

@ -0,0 +1,22 @@
1.1 EULA Addendum for faasd Community Edition (CE). This EULA Addendum for faasd is part of the [OpenFaaS Community Edition (CE) EULA](https://github.com/openfaas/faas/blob/master/EULA.md).
1.2 Agreement Parties. This Agreement is between OpenFaaS Ltd (the "Licensor") and you (the "Licensee").
1.3 Governing Law. This Agreement shall be governed by, and construed in accordance with, the laws of England and Wales.
1.4 OpenFaaS Edge (faasd-pro). OpenFaaS Edge (faasd-pro) is a separate commercial product that is fully licensed under the OpenFaaS Pro EULA.
2.1 Grant of License for faasd CE. faasd CE may be installed once per year for a single 60-day trial period in a commercial setting for the sole purpose of evaluation and testing. This trial does not include any support or warranty, and it terminates automatically at the end of the 60-day period unless a separate commercial license is obtained.
2.2 Personal (Non-Commercial) Use. faasd CE may be used by an individual for personal, non-commercial projects, provided that the user is not acting on behalf of any company, corporation, or other legal entity.
2.3 Restrictions.
(a) No redistribution, resale, or sublicensing of faasd CE is permitted.
(b) The License granted under this Addendum is non-transferable.
(c) Any continued commercial usage beyond the 60-day trial requires a separate commercial license from the Licensor.
2.4 Warranty Disclaimer. faasd CE is provided "as is," without warranty of any kind. No support or guarantee is included during the 60-day trial or for personal use.
2.5 Termination. If the terms of this Addendum are violated, the License granted hereunder terminates immediately. The Licensee must discontinue all use of faasd CE and destroy any copies in their possession.
2.6 Contact Information. For additional rights, commercial licenses, or support inquiries, please contact the Licensor at contact@openfaas.com.

View File

@ -1,8 +1,10 @@
License for faasd contributions from OpenFaaS Ltd - 2017, 2029-2024, see: EULA.md
Only third-party contributions to source code are licensed MIT:
MIT License
Copyright (c) 2020 Alex Ellis
Copyright (c) 2020 OpenFaaS Ltd
Copyright (c) 2020 OpenFaas Author(s)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

View File

@ -1,7 +1,7 @@
Version := $(shell git describe --tags --dirty)
GitCommit := $(shell git rev-parse HEAD)
LDFLAGS := "-s -w -X main.Version=$(Version) -X main.GitCommit=$(GitCommit)"
CONTAINERD_VER := 1.6.4
LDFLAGS := "-s -w -X github.com/openfaas/faasd/pkg.Version=$(Version) -X github.com/openfaas/faasd/pkg.GitCommit=$(GitCommit)"
CONTAINERD_VER := 1.7.27
CNI_VERSION := v0.9.1
ARCH := amd64
@ -20,11 +20,14 @@ local:
test:
CGO_ENABLED=0 GOOS=linux go test -mod=vendor -ldflags $(LDFLAGS) ./...
.PHONY: dist-local
dist-local:
CGO_ENABLED=0 GOOS=linux go build -mod=vendor -ldflags $(LDFLAGS) -o bin/faasd
.PHONY: dist
dist:
CGO_ENABLED=0 GOOS=linux go build -mod=vendor -ldflags $(LDFLAGS) -a -installsuffix cgo -o bin/faasd
CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -mod=vendor -ldflags $(LDFLAGS) -a -installsuffix cgo -o bin/faasd-armhf
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -mod=vendor -ldflags $(LDFLAGS) -a -installsuffix cgo -o bin/faasd-arm64
CGO_ENABLED=0 GOOS=linux go build -mod=vendor -ldflags $(LDFLAGS) -o bin/faasd
CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -mod=vendor -ldflags $(LDFLAGS) -o bin/faasd-arm64
.PHONY: hashgen
hashgen:
@ -33,7 +36,7 @@ hashgen:
.PHONY: prepare-test
prepare-test:
curl -sLSf https://github.com/containerd/containerd/releases/download/v$(CONTAINERD_VER)/containerd-$(CONTAINERD_VER)-linux-amd64.tar.gz > /tmp/containerd.tar.gz && sudo tar -xvf /tmp/containerd.tar.gz -C /usr/local/bin/ --strip-components=1
curl -SLfs https://raw.githubusercontent.com/containerd/containerd/v1.6.4/containerd.service | sudo tee /etc/systemd/system/containerd.service
curl -SLfs https://raw.githubusercontent.com/containerd/containerd/v1.7.0/containerd.service | sudo tee /etc/systemd/system/containerd.service
sudo systemctl daemon-reload && sudo systemctl start containerd
sudo /sbin/sysctl -w net.ipv4.conf.all.forwarding=1
sudo mkdir -p /opt/cni/bin
@ -66,3 +69,11 @@ test-e2e:
# Removed due to timing issue in CI on GitHub Actions
# /usr/local/bin/faas-cli logs figlet --since 15m --follow=false | grep Forking
verify-compose:
@echo Verifying docker-compose.yaml images in remote registries && \
arkade chart verify --verbose=$(VERBOSE) -f ./docker-compose.yaml
upgrade-compose:
@echo Checking for newer images in remote registries && \
arkade chart upgrade -f ./docker-compose.yaml --write

153
README.md
View File

@ -1,18 +1,38 @@
# faasd - a lightweight & portable faas engine
[![Sponsor this](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&link=https://github.com/sponsors/openfaas)](https://github.com/sponsors/openfaas)
[![Build Status](https://github.com/openfaas/faasd/workflows/build/badge.svg?branch=master)](https://github.com/openfaas/faasd/actions)
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
[![OpenFaaS](https://img.shields.io/badge/openfaas-serverless-blue.svg)](https://www.openfaas.com)
![Downloads](https://img.shields.io/github/downloads/openfaas/faasd/total)
# faasd - a lightweight and portable version of OpenFaaS
faasd is [OpenFaaS](https://github.com/openfaas/) reimagined, but without the cost and complexity of Kubernetes. It runs on a single host with very modest requirements, making it fast and easy to manage. Under the hood it uses [containerd](https://containerd.io/) and [Container Networking Interface (CNI)](https://github.com/containernetworking/cni) along with the same core OpenFaaS components from the main project.
![faasd logo](docs/media/social.png)
## Use-cases and tutorials
## Features & Benefits
faasd is just another way to run OpenFaaS, so many things you read in the docs or in blog posts will work the same way.
- **Lightweight** - faasd is a single Go binary, which runs as a systemd service making it easy to manage
- **Portable** - it runs on any Linux host with containerd and CNI, on as little as 2x vCPU and 2GB RAM - x86_64 and Arm64 supported
- **Easy to manage** - unlike Kubernetes, its API is stable and requires little maintenance
- **Low cost** - it's licensed per installation, so you can invoke your functions as much as you need, without additional cost
- **Stateful containers** - faasd supports stateful containers with persistent volumes such as PostgreSQL, Grafana, Prometheus, etc
- **Built on OpenFaaS** - uses the same containers that power OpenFaaS on Kubernetes for the Gateway, Queue-Worker, Event Connectors, Dashboards, Scale To Zero, etc
- **Ideal for internal business use** - use it to build internal tools, automate tasks, and integrate with existing systems
- **Deploy it for a customer** - package your functions along with OpenFaaS Edge into a VM image, and deploy it to your customers to run in their own datacenters
faasd does not create the same maintenance burden you'll find with installing, upgrading, and securing a Kubernetes cluster. You can deploy it and walk away, in the worst case, just deploy a new VM and deploy your functions again.
You can learn more about supported OpenFaaS features in the [ROADMAP.md](/docs/ROADMAP.md)
## Getting Started
There are two versions of faasd:
* faasd CE - for non-commercial, personal use only licensed under [the faasd CE EULA](/EULA.md)
* OpenFaaS Edge (faasd-pro) - fully licensed for commercial use
You can install either edition using the instructions in the [OpenFaaS docs](https://docs.openfaas.com/deployment/edge/).
You can request a license for [OpenFaaS Edge using this form](https://forms.gle/g6oKLTG29mDTSk5k9)
## Further resources
There are many blog posts and documentation pages about OpenFaaS on Kubernetes, which also apply to faasd.
Videos and overviews:
@ -30,63 +50,32 @@ Use-cases and tutorials:
Additional resources:
* The official handbook - [Serverless For Everyone Else](https://gumroad.com/l/serverless-for-everyone-else)
* The official handbook - [Serverless For Everyone Else](https://openfaas.gumroad.com/l/serverless-for-everyone-else)
* For reference: [OpenFaaS docs](https://docs.openfaas.com)
* For use-cases and tutorials: [OpenFaaS blog](https://openfaas.com/blog/)
* For self-paced learning: [OpenFaaS workshop](https://github.com/openfaas/workshop/)
### About faasd
### Deployment tutorials
* faasd is a static Golang binary
* uses the same core components and ecosystem of OpenFaaS
* uses containerd for its runtime and CNI for networking
* is multi-arch, so works on Intel `x86_64` and ARM out the box
* can run almost any other stateful container through its `docker-compose.yaml` file
* [Use multipass on Windows, MacOS or Linux](/docs/MULTIPASS.md)
* [Deploy to DigitalOcean with Terraform and TLS](https://www.openfaas.com/blog/faasd-tls-terraform/)
* [Deploy to any IaaS with cloud-init](https://blog.alexellis.io/deploy-serverless-faasd-with-cloud-init/)
* [Deploy faasd to your Raspberry Pi](https://blog.alexellis.io/faasd-for-lightweight-serverless/)
Most importantly, it's easy to manage so you can set it up and leave it alone to run your functions.
Terraform scripts:
[![demo](https://pbs.twimg.com/media/EPNQz00W4AEwDxM?format=jpg&name=medium)](https://www.youtube.com/watch?v=WX1tZoSXy8E)
* [Provision faasd on DigitalOcean with Terraform](docs/bootstrap/README.md)
* [Provision faasd with TLS on DigitalOcean with Terraform](docs/bootstrap/digitalocean-terraform/README.md)
> Demo of faasd running asynchronous functions
Watch the video: [faasd walk-through with cloud-init and Multipass](https://www.youtube.com/watch?v=WX1tZoSXy8E)
### Training / Handbook
### What does faasd deploy?
You can find various resources to learn about faasd for free, however the official handbook is the most comprehensive guide to getting started with faasd and OpenFaaS.
* faasd - itself, and its [faas-provider](https://github.com/openfaas/faas-provider) for containerd - CRUD for functions and services, implements the OpenFaaS REST API
* [Prometheus](https://github.com/prometheus/prometheus) - for monitoring of services, metrics, scaling and dashboards
* [OpenFaaS Gateway](https://github.com/openfaas/faas/tree/master/gateway) - the UI portal, CLI, and other OpenFaaS tooling can talk to this.
* [OpenFaaS queue-worker for NATS](https://github.com/openfaas/nats-queue-worker) - run your invocations in the background without adding any code. See also: [asynchronous invocations](https://docs.openfaas.com/reference/triggers/#async-nats-streaming)
* [NATS](https://nats.io) for asynchronous processing and queues
["Serverless For Everyone Else"](https://openfaas.gumroad.com/l/serverless-for-everyone-else) is the official handbook and was written to contribute funds towards the upkeep and maintenance of the project.
faasd relies on industry-standard tools for running containers:
* [CNI](https://github.com/containernetworking/plugins)
* [containerd](https://github.com/containerd/containerd)
* [runc](https://github.com/opencontainers/runc)
You can use the standard [faas-cli](https://github.com/openfaas/faas-cli) along with pre-packaged functions from *the Function Store*, or build your own using any OpenFaaS template.
### When should you use faasd over OpenFaaS on Kubernetes?
* To deploy microservices and functions that you can update and monitor remotely
* When you don't have the bandwidth to learn or manage Kubernetes
* To deploy embedded apps in IoT and edge use-cases
* To distribute applications to a customer or client
* You have a cost sensitive project - run faasd on a 1GB VM for 5-10 USD / mo or on your Raspberry Pi
* When you just need a few functions or microservices, without the cost of a cluster
faasd does not create the same maintenance burden you'll find with maintaining, upgrading, and securing a Kubernetes cluster. You can deploy it and walk away, in the worst case, just deploy a new VM and deploy your functions again.
## Learning faasd
The faasd project is MIT licensed and open source, and you will find some documentation, blog posts and videos for free.
However, "Serverless For Everyone Else" is the official handbook and was written to contribute funds towards the upkeep and maintenance of the project.
### The official handbook and docs for faasd
<a href="https://gumroad.com/l/serverless-for-everyone-else">
<a href="https://openfaas.gumroad.com/l/serverless-for-everyone-else">
<img src="https://www.alexellis.io/serverless.png" width="40%"></a>
You'll learn how to deploy code in any language, lift and shift Dockerfiles, run requests in queues, write background jobs and to integrate with databases. faasd packages the same code as OpenFaaS, so you get built-in metrics for your HTTP endpoints, a user-friendly CLI, pre-packaged functions and templates from the store and a UI.
@ -113,60 +102,4 @@ Topics include:
View sample pages, reviews and testimonials on Gumroad:
["Serverless For Everyone Else"](https://gumroad.com/l/serverless-for-everyone-else)
### Deploy faasd
The easiest way to deploy faasd is with cloud-init, we give several examples below, and post IaaS platforms will accept "user-data" pasted into their UI, or via their API.
For trying it out on MacOS or Windows, we recommend using [multipass](https://multipass.run) to run faasd in a VM.
If you don't use cloud-init, or have already created your Linux server you can use the installation script as per below:
```bash
git clone https://github.com/openfaas/faasd --depth=1
cd faasd
./hack/install.sh
```
> This approach also works for Raspberry Pi
It's recommended that you do not install Docker on the same host as faasd, since 1) they may both use different versions of containerd and 2) docker's networking rules can disrupt faasd's networking. When using faasd - make your faasd server a faasd server, and build container image on your laptop or in a CI pipeline.
#### Deployment tutorials
* [Use multipass on Windows, MacOS or Linux](/docs/MULTIPASS.md)
* [Deploy to DigitalOcean with Terraform and TLS](https://www.openfaas.com/blog/faasd-tls-terraform/)
* [Deploy to any IaaS with cloud-init](https://blog.alexellis.io/deploy-serverless-faasd-with-cloud-init/)
* [Deploy faasd to your Raspberry Pi](https://blog.alexellis.io/faasd-for-lightweight-serverless/)
Terraform scripts:
* [Provision faasd on DigitalOcean with Terraform](docs/bootstrap/README.md)
* [Provision faasd with TLS on DigitalOcean with Terraform](docs/bootstrap/digitalocean-terraform/README.md)
### Function and template store
For community functions see `faas-cli store --help`
For templates built by the community see: `faas-cli template store list`, you can also use the `dockerfile` template if you just want to migrate an existing service without the benefits of using a template.
### Community support
Commercial users and solo business owners should become OpenFaaS GitHub Sponsors to receive regular email updates on changes, tutorials and new features.
If you are learning faasd, or want to share your use-case, you can join the OpenFaaS Slack community.
* [Become an OpenFaaS GitHub Sponsor](https://github.com/sponsors/openfaas/)
* [Join Slack](https://slack.openfaas.io/)
### Backlog, features and known issues
For completed features, WIP and upcoming roadmap see:
See [ROADMAP.md](docs/ROADMAP.md)
Want to build a patch without setting up a complete development environment? See [docs/PATCHES.md](docs/PATCHES.md)
Are you looking to hack on faasd? Follow the [developer instructions](docs/DEV.md) for a manual installation, or use the `hack/install.sh` script and pick up from there.
["Serverless For Everyone Else"](https://openfaas.gumroad.com/l/serverless-for-everyone-else)

View File

@ -50,54 +50,52 @@ func runInstall(_ *cobra.Command, _ []string) error {
return err
}
err := binExists("/usr/local/bin/", "faasd")
if err != nil {
if err := binExists("/usr/local/bin/", "faasd"); err != nil {
return err
}
err = systemd.InstallUnit("faasd-provider", map[string]string{
if err := systemd.InstallUnit("faasd-provider", map[string]string{
"Cwd": faasdProviderWd,
"SecretMountPath": path.Join(faasdwd, "secrets")})
if err != nil {
"SecretMountPath": path.Join(faasdwd, "secrets")}); err != nil {
return err
}
err = systemd.InstallUnit("faasd", map[string]string{"Cwd": faasdwd})
if err != nil {
if err := systemd.InstallUnit("faasd", map[string]string{"Cwd": faasdwd}); err != nil {
return err
}
err = systemd.DaemonReload()
if err != nil {
if err := systemd.DaemonReload(); err != nil {
return err
}
err = systemd.Enable("faasd-provider")
if err != nil {
if err := systemd.Enable("faasd-provider"); err != nil {
return err
}
err = systemd.Enable("faasd")
if err != nil {
if err := systemd.Enable("faasd"); err != nil {
return err
}
err = systemd.Start("faasd-provider")
if err != nil {
if err := systemd.Start("faasd-provider"); err != nil {
return err
}
err = systemd.Start("faasd")
if err != nil {
if err := systemd.Start("faasd"); err != nil {
return err
}
fmt.Println(`Check status with:
fmt.Println(`
The initial setup downloads various container images, which may take a
minute or two depending on your connection.
Check the status of the faasd service with:
sudo journalctl -u faasd --lines 100 -f
Login with:
sudo cat /var/lib/faasd/secrets/basic-auth-password | faas-cli login -s`)
sudo -E cat /var/lib/faasd/secrets/basic-auth-password | faas-cli login -s`)
fmt.Println("")
return nil
}

View File

@ -3,7 +3,6 @@ package cmd
import (
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
@ -30,19 +29,13 @@ func makeProviderCmd() *cobra.Command {
Short: "Run the faasd-provider",
}
command.Flags().String("pull-policy", "Always", `Set to "Always" to force a pull of images upon deployment, or "IfNotPresent" to try to use a cached image.`)
command.RunE = runProviderE
command.PreRunE = preRunE
command.RunE = func(_ *cobra.Command, _ []string) error {
return command
}
pullPolicy, flagErr := command.Flags().GetString("pull-policy")
if flagErr != nil {
return flagErr
}
alwaysPull := false
if pullPolicy == "Always" {
alwaysPull = true
}
func runProviderE(cmd *cobra.Command, _ []string) error {
config, providerConfig, err := config.ReadFromEnv(types.OsEnv{})
if err != nil {
@ -57,18 +50,15 @@ func makeProviderCmd() *cobra.Command {
return err
}
writeHostsErr := ioutil.WriteFile(path.Join(wd, "hosts"),
[]byte(`127.0.0.1 localhost`), workingDirectoryPermission)
if writeHostsErr != nil {
return fmt.Errorf("cannot write hosts file: %s", writeHostsErr)
if err := os.WriteFile(path.Join(wd, "hosts"),
[]byte(`127.0.0.1 localhost`), workingDirectoryPermission); err != nil {
return fmt.Errorf("cannot write hosts file: %s", err)
}
writeResolvErr := ioutil.WriteFile(path.Join(wd, "resolv.conf"),
[]byte(`nameserver 8.8.8.8`), workingDirectoryPermission)
if writeResolvErr != nil {
return fmt.Errorf("cannot write resolv.conf file: %s", writeResolvErr)
if err := os.WriteFile(path.Join(wd, "resolv.conf"),
[]byte(`nameserver 8.8.8.8
nameserver 8.8.4.4`), workingDirectoryPermission); err != nil {
return fmt.Errorf("cannot write resolv.conf file: %s", err)
}
cni, err := cninetwork.InitNetwork()
@ -92,27 +82,26 @@ func makeProviderCmd() *cobra.Command {
return err
}
alwaysPull := true
bootstrapHandlers := types.FaaSHandlers{
FunctionProxy: proxy.NewHandlerFunc(*config, invokeResolver),
DeleteHandler: handlers.MakeDeleteHandler(client, cni),
DeployHandler: handlers.MakeDeployHandler(client, cni, baseUserSecretsPath, alwaysPull),
FunctionReader: handlers.MakeReadHandler(client),
ReplicaReader: handlers.MakeReplicaReaderHandler(client),
ReplicaUpdater: handlers.MakeReplicaUpdateHandler(client, cni),
UpdateHandler: handlers.MakeUpdateHandler(client, cni, baseUserSecretsPath, alwaysPull),
HealthHandler: func(w http.ResponseWriter, r *http.Request) {},
InfoHandler: handlers.MakeInfoHandler(Version, GitCommit),
ListNamespaceHandler: handlers.MakeNamespacesLister(client),
SecretHandler: handlers.MakeSecretHandler(client.NamespaceService(), baseUserSecretsPath),
LogHandler: logs.NewLogHandlerFunc(faasdlogs.New(), config.ReadTimeout),
FunctionProxy: httpHeaderMiddleware(proxy.NewHandlerFunc(*config, invokeResolver, false)),
DeleteFunction: httpHeaderMiddleware(handlers.MakeDeleteHandler(client, cni)),
DeployFunction: httpHeaderMiddleware(handlers.MakeDeployHandler(client, cni, baseUserSecretsPath, alwaysPull)),
FunctionLister: httpHeaderMiddleware(handlers.MakeReadHandler(client)),
FunctionStatus: httpHeaderMiddleware(handlers.MakeReplicaReaderHandler(client)),
ScaleFunction: httpHeaderMiddleware(handlers.MakeReplicaUpdateHandler(client, cni)),
UpdateFunction: httpHeaderMiddleware(handlers.MakeUpdateHandler(client, cni, baseUserSecretsPath, alwaysPull)),
Health: httpHeaderMiddleware(func(w http.ResponseWriter, r *http.Request) {}),
Info: httpHeaderMiddleware(handlers.MakeInfoHandler(faasd.Version, faasd.GitCommit)),
ListNamespaces: httpHeaderMiddleware(handlers.MakeNamespacesLister(client)),
Secrets: httpHeaderMiddleware(handlers.MakeSecretHandler(client.NamespaceService(), baseUserSecretsPath)),
Logs: httpHeaderMiddleware(logs.NewLogHandlerFunc(faasdlogs.New(), config.ReadTimeout)),
MutateNamespace: httpHeaderMiddleware(handlers.MakeMutateNamespace(client)),
}
log.Printf("Listening on TCP port: %d\n", *config.TCPPort)
bootstrap.Serve(&bootstrapHandlers, config)
log.Printf("Listening on: 0.0.0.0:%d", *config.TCPPort)
bootstrap.Serve(cmd.Context(), &bootstrapHandlers, config)
return nil
}
return command
}
/*
@ -127,7 +116,7 @@ func moveSecretsToDefaultNamespaceSecrets(baseSecretPath string, defaultNamespac
return err
}
files, err := ioutil.ReadDir(baseSecretPath)
files, err := os.ReadDir(baseSecretPath)
if err != nil {
return err
}
@ -174,3 +163,10 @@ func copyFile(src, dst string) error {
return nil
}
func httpHeaderMiddleware(next http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-OpenFaaS-EULA", "openfaas-ce")
next.ServeHTTP(w, r)
}
}

View File

@ -4,6 +4,7 @@ import (
"fmt"
"github.com/morikuni/aec"
"github.com/openfaas/faasd/pkg"
"github.com/spf13/cobra"
)
@ -16,25 +17,15 @@ func init() {
rootCommand.AddCommand(installCmd)
rootCommand.AddCommand(makeProviderCmd())
rootCommand.AddCommand(collectCmd)
rootCommand.AddCommand(makeServiceCmd())
}
func RootCommand() *cobra.Command {
return rootCommand
}
var (
// GitCommit Git Commit SHA
GitCommit string
// Version version of the CLI
Version string
)
// Execute faasd
func Execute(version, gitCommit string) error {
// Get Version and GitCommit values from main.go.
Version = version
GitCommit = gitCommit
func Execute() error {
if err := rootCommand.Execute(); err != nil {
return err
@ -46,12 +37,16 @@ var rootCommand = &cobra.Command{
Use: "faasd",
Short: "Start faasd",
Long: `
faasd - Serverless For Everyone Else
faasd Community Edition (CE):
Learn how to build, secure, and monitor functions with faasd with
the eBook:
https://gumroad.com/l/serverless-for-everyone-else
https://openfaas.gumroad.com/l/serverless-for-everyone-else
License: OpenFaaS CE EULA with faasd addendum:
https://github.com/openfaas/faasd/blob/master/EULA.md
`,
RunE: runRootCommand,
SilenceUsage: true,
@ -78,7 +73,7 @@ func parseBaseCommand(_ *cobra.Command, _ []string) {
}
func printVersion() {
fmt.Printf("faasd version: %s\tcommit: %s\n", GetVersion(), GitCommit)
fmt.Printf("faasd Community Edition (CE) version: %s\tcommit: %s\n", pkg.GetVersion(), pkg.GitCommit)
}
func printLogo() {
@ -86,14 +81,6 @@ func printLogo() {
fmt.Println(logoText)
}
// GetVersion get latest version
func GetVersion() string {
if len(Version) == 0 {
return "dev"
}
return Version
}
// Logo for version and root command
const Logo = ` __ _
/ _| __ _ __ _ ___ __| |

22
cmd/service.go Normal file
View File

@ -0,0 +1,22 @@
package cmd
import "github.com/spf13/cobra"
func makeServiceCmd() *cobra.Command {
var command = &cobra.Command{
Use: "service",
Short: "Manage services",
Long: `Manage services created by faasd from the docker-compose.yml file`,
}
command.RunE = runServiceE
command.AddCommand(makeServiceLogsCmd())
return command
}
func runServiceE(cmd *cobra.Command, args []string) error {
return cmd.Help()
}

89
cmd/service_logs.go Normal file
View File

@ -0,0 +1,89 @@
package cmd
import (
"context"
"errors"
"fmt"
"os"
"time"
goexecute "github.com/alexellis/go-execute/v2"
"github.com/spf13/cobra"
)
func makeServiceLogsCmd() *cobra.Command {
var command = &cobra.Command{
Use: "logs",
Short: "View logs for a service",
Long: `View logs for a service created by faasd from the docker-compose.yml file.`,
Example: ` ## View logs for the gateway for the last hour
faasd service logs gateway --since 1h
## View logs for the cron-connector, and tail them
faasd service logs cron-connector -f
`,
}
command.Flags().Duration("since", 10*time.Minute, "How far back in time to include logs")
command.Flags().BoolP("follow", "f", false, "Follow the logs")
command.RunE = runServiceLogsE
command.PreRunE = preRunServiceLogsE
return command
}
func runServiceLogsE(cmd *cobra.Command, args []string) error {
name := args[0]
namespace, _ := cmd.Flags().GetString("namespace")
follow, _ := cmd.Flags().GetBool("follow")
since, _ := cmd.Flags().GetDuration("since")
journalTask := goexecute.ExecTask{
Command: "journalctl",
Args: []string{"-o", "cat", "-t", fmt.Sprintf("%s:%s", namespace, name)},
StreamStdio: true,
}
if follow {
journalTask.Args = append(journalTask.Args, "-f")
}
if since != 0 {
// Calculate the timestamp that is 'age' duration ago
sinceTime := time.Now().Add(-since)
// Format according to journalctl's expected format: "2012-10-30 18:17:16"
formattedTime := sinceTime.Format("2006-01-02 15:04:05")
journalTask.Args = append(journalTask.Args, fmt.Sprintf("--since=%s", formattedTime))
}
res, err := journalTask.Execute(context.Background())
if err != nil {
return err
}
if res.ExitCode != 0 {
return fmt.Errorf("failed to get logs for service %s: %s", name, res.Stderr)
}
return nil
}
func preRunServiceLogsE(cmd *cobra.Command, args []string) error {
if os.Geteuid() != 0 {
return errors.New("this command must be run as root")
}
if len(args) == 0 {
return errors.New("service name is required as an argument")
}
namespace, _ := cmd.Flags().GetString("namespace")
if namespace == "" {
return errors.New("namespace is required")
}
return nil
}

View File

@ -2,7 +2,6 @@ package cmd
import (
"fmt"
"io/ioutil"
"log"
"os"
"os/signal"
@ -42,6 +41,7 @@ var upCmd = &cobra.Command{
Use: "up",
Short: "Start faasd",
RunE: runUp,
PreRunE: preRunE,
}
func runUp(cmd *cobra.Command, _ []string) error {
@ -166,7 +166,7 @@ func makeFile(filePath, fileContents string) error {
return nil
} else if os.IsNotExist(err) {
log.Printf("Writing to: %q\n", filePath)
return ioutil.WriteFile(filePath, []byte(fileContents), workingDirectoryPermission)
return os.WriteFile(filePath, []byte(fileContents), workingDirectoryPermission)
} else {
return err
}
@ -204,3 +204,11 @@ func parseUpFlags(cmd *cobra.Command) (upConfig, error) {
parsed.workingDir = faasdwd
return parsed, err
}
func preRunE(cmd *cobra.Command, _ []string) error {
if err := pkg.ConnectivityCheck(); err != nil {
return fmt.Errorf("the OpenFaaS CE EULA requires Internet access, upgrade to faasd Pro to continue")
}
return nil
}

View File

@ -1,25 +1,8 @@
version: "3.7"
services:
basic-auth-plugin:
image: ghcr.io/openfaas/basic-auth:0.21.4
environment:
- port=8080
- secret_mount_path=/run/secrets
- user_filename=basic-auth-user
- pass_filename=basic-auth-password
volumes:
# we assume cwd == /var/lib/faasd
- type: bind
source: ./secrets/basic-auth-password
target: /run/secrets/basic-auth-password
- type: bind
source: ./secrets/basic-auth-user
target: /run/secrets/basic-auth-user
cap_add:
- CAP_NET_RAW
nats:
image: docker.io/library/nats-streaming:0.22.0
image: docker.io/library/nats-streaming:0.25.6
# nobody
user: "65534"
command:
@ -38,7 +21,7 @@ services:
# - "127.0.0.1:8222:8222"
prometheus:
image: docker.io/prom/prometheus:v2.14.0
image: docker.io/prom/prometheus:v3.1.0
# nobody
user: "65534"
volumes:
@ -56,7 +39,7 @@ services:
- "127.0.0.1:9090:9090"
gateway:
image: ghcr.io/openfaas/gateway:0.22.0
image: ghcr.io/openfaas/gateway:0.27.12
environment:
- basic_auth=true
- functions_provider_url=http://faasd-provider:8081/
@ -66,8 +49,6 @@ services:
- upstream_timeout=65s
- faas_nats_address=nats
- faas_nats_port=4222
- auth_proxy_url=http://basic-auth-plugin:8080/validate
- auth_proxy_pass_body=false
- secret_mount_path=/run/secrets
- scale_from_zero=true
- function_namespace=openfaas-fn
@ -82,14 +63,13 @@ services:
cap_add:
- CAP_NET_RAW
depends_on:
- basic-auth-plugin
- nats
- prometheus
ports:
- "8080:8080"
queue-worker:
image: ghcr.io/openfaas/queue-worker:0.12.2
image: ghcr.io/openfaas/queue-worker:0.14.2
environment:
- faas_nats_address=nats
- faas_nats_port=4222

View File

@ -28,7 +28,7 @@ If you're using multipass, then allocate sufficient resources:
```bash
multipass launch \
--mem 4G \
--memory 4G \
-c 2 \
-n faasd
@ -57,7 +57,6 @@ curl -sLS https://cli.openfaas.com | sudo sh
#### Install the CNI plugins:
* For PC run `export ARCH=amd64`
* For RPi/armhf run `export ARCH=arm`
* For arm64 run `export ARCH=arm64`
Then run:
@ -83,26 +82,17 @@ EOF'
### Get containerd
You have three options - binaries for PC, binaries for armhf, or build from source.
* Install containerd `x86_64` only
```bash
export VER=1.6.4
curl -sSL https://github.com/containerd/containerd/releases/download/v$VER/containerd-$VER-linux-amd64.tar.gz > /tmp/containerd.tar.gz \
export VER=1.7.27
curl -sSL https://github.com/containerd/containerd/releases/download/v$VER/containerd-$VER-linux-amd64.tar.gz -o /tmp/containerd.tar.gz \
&& sudo tar -xvf /tmp/containerd.tar.gz -C /usr/local/bin/ --strip-components=1
containerd -version
```
* Or get my containerd binaries for Raspberry Pi (armhf)
Building `containerd` on armhf is extremely slow, so I've provided binaries for you.
```bash
curl -sSL https://github.com/alexellis/containerd-armhf/releases/download/v1.6.4/containerd.tgz | sudo tar -xvz --strip-components=2 -C /usr/local/bin/
```
* Or clone / build / install [containerd](https://github.com/containerd/containerd) from source:
```bash
@ -112,7 +102,7 @@ containerd -version
git clone https://github.com/containerd/containerd
cd containerd
git fetch origin --tags
git checkout v1.6.4
git checkout v1.7.27
make
sudo make install
@ -123,7 +113,7 @@ containerd -version
#### Ensure containerd is running
```bash
curl -sLS https://raw.githubusercontent.com/containerd/containerd/v1.6.4/containerd.service > /tmp/containerd.service
curl -sLS https://raw.githubusercontent.com/containerd/containerd/v1.7.27/containerd.service > /tmp/containerd.service
# Extend the timeouts for low-performance VMs
echo "[Manager]" | tee -a /tmp/containerd.service
@ -230,14 +220,11 @@ sudo cp bin/faasd /usr/local/bin
# For x86_64
export SUFFIX=""
# armhf
export SUFFIX="-armhf"
# arm64
export SUFFIX="-arm64"
# Then download
curl -fSLs "https://github.com/openfaas/faasd/releases/download/0.13.0/faasd$SUFFIX" \
curl -fSLs "https://github.com/openfaas/faasd/releases/download/0.16.2/faasd$SUFFIX" \
-o "/tmp/faasd" \
&& chmod +x "/tmp/faasd"
sudo mv /tmp/faasd /usr/local/bin/
@ -253,7 +240,7 @@ sudo faasd install
2020/02/17 17:38:06 Writing to: "/var/lib/faasd/secrets/basic-auth-password"
2020/02/17 17:38:06 Writing to: "/var/lib/faasd/secrets/basic-auth-user"
Login with:
sudo cat /var/lib/faasd/secrets/basic-auth-password | faas-cli login -s
sudo -E cat /var/lib/faasd/secrets/basic-auth-password | faas-cli login -s
```
You can now log in either from this machine or a remote machine using the OpenFaaS UI, or CLI.

View File

@ -40,7 +40,7 @@ It took me about 2-3 minutes to run through everything after installing multipas
sed "s/ssh-rsa.*/$(cat $HOME/.ssh/id_*.pub)/" cloud-config.txt | multipass launch --name faasd --cloud-init -
```
This can also be done manually, just replace the 2nd line of the `cloud-config.txt` with the coPntents of your public ssh key, usually either `~/.ssh/id_rsa.pub` or `~/.ssh/id_ed25519.pub`
This can also be done manually, just replace the 2nd line of the `cloud-config.txt` with the contents of your public ssh key, usually either `~/.ssh/id_rsa.pub` or `~/.ssh/id_ed25519.pub`
```
ssh_authorized_keys:

View File

@ -12,7 +12,7 @@ Your SSH key will be used, so that you can copy a new faasd binary over to the h
```bash
multipass launch \
--mem 4G \
--memory 4G \
-c 2 \
-n faasd
@ -84,5 +84,5 @@ See the testing instructions on the PR and run through those steps.
Post your results on GitHub to assist the creator of the pull request.
You can see how to get the logs for various components using the [eBook Serverless For Everyone Else](https://gumroad.com/l/serverless-for-everyone-else), or by consulting the [DEV.md](/docs/DEV.md) guide.
You can see how to get the logs for various components using the [eBook Serverless For Everyone Else](https://openfaas.gumroad.com/l/serverless-for-everyone-else), or by consulting the [DEV.md](/docs/DEV.md) guide.

View File

@ -1,55 +1,79 @@
# faasd backlog and features
It's important to understand the vision and tradeoffs between OpenFaaS Edge (faasd-pro) vs. OpenFaaS on Kubernetes.
faasd is a single-node implementation of OpenFaaS.
It is supposed to be a lightweight, low-overhead, way to deploy OpenFaaS functions for functions which do not need to scale out.
It is not supposed to have multiple replicas, clustering, High Availability (HA), or auto-scaling.
[Learn when to use faasd](https://docs.openfaas.com/deployment/)
## Supported operations
* `faas login`
* `faas up`
* `faas list`
* `faas describe`
* `faas deploy --update=true --replace=false`
* `faas invoke --async`
* `faas invoke`
* `faas rm`
* `faas store list/deploy/inspect`
* `faas version`
* `faas namespace`
* `faas secret`
* `faas logs`
* `faas auth` - supported for Basic Authentication and OpenFaaS PRO with OIDC and Single-sign On.
* `faas-cli login`
* `faas-cli up`
* `faas-cli list`
* `faas-cli describe`
* `faas-cli deploy --update=true --replace=false`
* `faas-cli invoke --async`
* `faas-cli invoke`
* `faas-cli rm`
* `faas-cli store list/deploy/inspect`
* `faas-cli version`
* `faas-cli namespace`
* `faas-cli secret`
* `faas-cli logs`
* `faas-cli auth` - supported for Basic Authentication and OpenFaaS Pro with OIDC and Single-sign On.
Scale from and to zero is also supported. On a Dell XPS with a small, pre-pulled image unpausing an existing task took 0.19s and starting a task for a killed function took 0.39s. There may be further optimizations to be gained.
The OpenFaaS REST API is supported by faasd, learn more in the [manual](https://store.openfaas.com/l/serverless-for-everyone-else) under "Can I get an API with that?"
## Constraints vs OpenFaaS on Kubernetes
faasd suits certain use-cases as mentioned in the README file, for those who want a solution which can scale out horizontally with minimum effort, Kubernetes or K3s is a valid option.
faasd suits certain use-cases as mentioned in the [README.md](/README.md) file, for those who want a solution which can scale out horizontally with minimum effort, Kubernetes or K3s is a valid option.
Which is right for you? [Read a comparison in the OpenFaaS docs](https://docs.openfaas.com/deployment/)
### One replica per function
Functions only support one replica, so cannot scale horizontally, but can scale vertically.
Functions only support one replica for each function, so that means horizontal scaling is not available.
Workaround: deploy one uniquely named function per replica.
It can scale vertically, and this may be a suitable alternative for many use-cases. See the [YAML reference for how to configure limits](https://docs.openfaas.com/reference/yaml/).
### Scale from zero may give a non-200
Workaround: deploy multiple, dynamically named functions `scraper-1`, `scraper-2`, `scraper-3` and set up a reverse proxy rule to load balance i.e. `scraper.example.com => [/function/scraper-1, /function/scraper-2, /function/scraper-3]`.
When scaling from zero there is no health check implemented, so the request may arrive before your HTTP server is ready to serve a request, and therefore give a non-200 code.
### Leaf-node only - no clustering
Workaround: Do not scale to zero, or have your client retry HTTP calls.
faasd is operates on a leaf-node/single-node model. If this is an issue for you, but you have resource constraints, you will need to use [OpenFaaS CE or Pro on Kubernetes](https://docs.openfaas.com/deployment/).
### No clustering is available
There are no plans to add any form of clustering or multi-node support to faasd.
No clustering is available in faasd, however you can still apply fault-tolerance and high availability techniques.
See past discussion at: [HA / resilience in faasd #225](https://github.com/openfaas/faasd/issues/225)
Workaround: deploy multiple faasd instances and use a hardware or software load-balancer. Take regular VM/host snapshots or backups.
What about HA and fault tolerance?
### No rolling updates
To achieve fault tolerance, you could put two faasd instances behind a load balancer or proxy, but you will need to deploy the same set of functions to each.
When running `faas-cli deploy`, your old function is removed before the new one is started. This may cause a small amount of downtime, depending on the timeouts and grace periods you set.
An alternative would be to take regular VM backups or snapshots.
Workaround: deploy uniquely named functions per version, and switch an Ingress or Reverse Proxy record to point at a new version once it is ready.
### No rolling updates are available today
When running `faas-cli deploy`, your old function is removed before the new one is started. This may cause a period of downtime, depending on the timeouts and grace periods you set.
Workaround: deploy uniquely named functions i.e. `scraper-1` and `scraper-2` with a reverse proxy rule that maps `/function/scraper` to the active version.
## Known issues
### Non 200 HTTP status from the gateway upon first use
### Troubleshooting
There is a very detailed chapter on troubleshooting in the eBook [Serverless For Everyone Else](https://store.openfaas.com/l/serverless-for-everyone-else)
### Your function timed-out at 60 seconds
See the manual for how to configure a longer timeouts.
### Non 200 HTTP status from the gateway upon reboot
This issue appears to happen sporadically and only for some users.
@ -75,16 +99,18 @@ sudo systemctl restart faasd
Should have:
* [ ] Offer a recommendation or implement a strategy for faasd replication/HA
* [ ] Monitor and restart any of the core components at runtime if the container stops
* [ ] Asynchronous function deletion instead of synchronous
* [ ] Asynchronous function start-up instead of synchronous
* [ ] Restart any of the containers in docker-compose.yaml if they crash.
* [ ] Asynchronous function deployment and deletion (currently synchronous/blocking)
Nice to Have:
* [ ] Terraform for AWS (in-progress)
* [ ] Total memory limits - if a node has 1GB of RAM, don't allow more than 1000MB of RAM to be reserved via limits
* [ ] Offer live rolling-updates, with zero downtime - requires moving to IDs vs. names for function containers
* [ ] Live rolling-updates, with zero downtime (may require using IDs instead of names for function containers)
* [ ] Apply a total memory limit for the host (if a node has 1GB of RAM, don't allow more than 1GB of RAM to be specified in the limits field)
* [ ] Terraform for AWS EC2
Won't have:
* [ ] Clustering
* [ ] Multiple replicas per function
### Completed
@ -103,7 +129,7 @@ Nice to Have:
* [x] Self-install / create systemd service via `faasd install`
* [x] Restart containers upon restart of faasd
* [x] Clear / remove containers and tasks with SIGTERM / SIGINT
* [x] Determine armhf/arm64 containers to run for gateway
* [x] Determine arm64 containers to run for gateway
* [x] Configure `basic_auth` to protect the OpenFaaS gateway and faasd-provider HTTP API
* [x] Setup custom working directory for faasd `/var/lib/faasd/`
* [x] Use CNI to create network namespaces and adapters
@ -115,4 +141,7 @@ Nice to Have:
* [x] Terraform for DigitalOcean
* [x] [Store and retrieve annotations in function spec](https://github.com/openfaas/faasd/pull/86) - in progress
* [x] An installer for faasd and dependencies - runc, containerd
* [x] Offer a recommendation or implement a strategy for faasd replication/HA
* [x] [Remove / deprecate armhf / armv7 support](https://github.com/openfaas/faasd/issues/364)
* [x] Add support for CPU/RAM metrics in the UI, CLI and API
* [x] Network segmentation (functions cannot talk to each other or the host)

View File

@ -6,8 +6,8 @@ packages:
- git
runcmd:
- curl -sLSf https://github.com/containerd/containerd/releases/download/v1.6.4/containerd-1.6.4-linux-amd64.tar.gz > /tmp/containerd.tar.gz && tar -xvf /tmp/containerd.tar.gz -C /usr/local/bin/ --strip-components=1
- curl -SLfs https://raw.githubusercontent.com/containerd/containerd/v1.6.4/containerd.service | tee /etc/systemd/system/containerd.service
- curl -sLSf https://github.com/containerd/containerd/releases/download/v1.7.0/containerd-1.7.0-linux-amd64.tar.gz > /tmp/containerd.tar.gz && tar -xvf /tmp/containerd.tar.gz -C /usr/local/bin/ --strip-components=1
- curl -SLfs https://raw.githubusercontent.com/containerd/containerd/v1.7.0/containerd.service | tee /etc/systemd/system/containerd.service
- systemctl daemon-reload && systemctl start containerd
- /sbin/sysctl -w net.ipv4.conf.all.forwarding=1
- mkdir -p /opt/cni/bin
@ -16,8 +16,8 @@ runcmd:
- mkdir -p /var/lib/faasd/secrets/
- echo ${gw_password} > /var/lib/faasd/secrets/basic-auth-password
- echo admin > /var/lib/faasd/secrets/basic-auth-user
- cd /go/src/github.com/openfaas/ && git clone --depth 1 --branch 0.13.0 https://github.com/openfaas/faasd
- curl -fSLs "https://github.com/openfaas/faasd/releases/download/0.13.0/faasd" --output "/usr/local/bin/faasd" && chmod a+x "/usr/local/bin/faasd"
- cd /go/src/github.com/openfaas/ && git clone --depth 1 --branch 0.16.2 https://github.com/openfaas/faasd
- curl -fSLs "https://github.com/openfaas/faasd/releases/download/0.16.2/faasd" --output "/usr/local/bin/faasd" && chmod a+x "/usr/local/bin/faasd"
- cd /go/src/github.com/openfaas/faasd/ && /usr/local/bin/faasd install
- systemctl status -l containerd --no-pager
- journalctl -u faasd-provider --no-pager

129
go.mod
View File

@ -1,80 +1,101 @@
module github.com/openfaas/faasd
go 1.18
go 1.23.0
toolchain go1.23.7
require (
github.com/alexellis/go-execute v0.5.0
github.com/alexellis/k3sup v0.0.0-20220105194923-e2bb18116d36
github.com/alexellis/arkade v0.0.0-20240320084407-6cf4a641c415
github.com/compose-spec/compose-go v0.0.0-20200528042322-36d8ce368e05
github.com/containerd/containerd v1.6.4
github.com/containerd/go-cni v1.1.5
github.com/containerd/containerd v1.7.27
github.com/containerd/go-cni v1.1.9
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
github.com/docker/cli v0.0.0-20191105005515-99c5edceb48d
github.com/docker/distribution v2.8.1+incompatible
github.com/docker/docker v17.12.0-ce-rc1.0.20191113042239-ea84732a7725+incompatible // indirect
github.com/docker/docker-credential-helpers v0.6.3 // indirect
github.com/docker/go-units v0.4.0
github.com/gorilla/mux v1.8.0
github.com/docker/cli v24.0.7+incompatible
github.com/docker/docker v24.0.7+incompatible // indirect
github.com/docker/go-units v0.5.0
github.com/gorilla/mux v1.8.1
github.com/morikuni/aec v1.0.0
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
github.com/openfaas/faas-provider v0.19.0
github.com/openfaas/faas/gateway v0.0.0-20220509091830-4e868f5f9d81
github.com/opencontainers/runtime-spec v1.1.0
github.com/openfaas/faas-provider v0.25.3
github.com/pkg/errors v0.9.1
github.com/sethvargo/go-password v0.2.0
github.com/spf13/cobra v1.4.0
github.com/spf13/cobra v1.8.0
github.com/spf13/pflag v1.0.5
github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f
golang.org/x/sys v0.0.0-20220702020025-31831981b65f
k8s.io/apimachinery v0.24.0
github.com/vishvananda/netlink v1.2.1-beta.2
github.com/vishvananda/netns v0.0.4
golang.org/x/sys v0.31.0
k8s.io/apimachinery v0.29.3
)
require (
github.com/Microsoft/go-winio v0.5.1 // indirect
github.com/Microsoft/hcsshim v0.9.2 // indirect
github.com/alexellis/go-execute/v2 v2.2.1
github.com/distribution/reference v0.6.0
)
require (
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/Microsoft/hcsshim v0.11.7 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/containerd/cgroups v1.0.3 // indirect
github.com/containerd/continuity v0.2.2 // indirect
github.com/containerd/fifo v1.0.0 // indirect
github.com/containerd/ttrpc v1.1.0 // indirect
github.com/containerd/typeurl v1.0.2 // indirect
github.com/containernetworking/cni v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/containerd/cgroups v1.1.0 // indirect
github.com/containerd/containerd/api v1.8.0 // indirect
github.com/containerd/continuity v0.4.4 // indirect
github.com/containerd/errdefs v0.3.0 // indirect
github.com/containerd/fifo v1.1.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/platforms v0.2.1 // indirect
github.com/containerd/ttrpc v1.2.7 // indirect
github.com/containerd/typeurl/v2 v2.1.1 // indirect
github.com/containernetworking/cni v1.1.2 // indirect
github.com/docker/docker-credential-helpers v0.8.0 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
github.com/gogo/googleapis v1.4.0 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/uuid v1.2.0 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/klauspost/compress v1.11.13 // indirect
github.com/mattn/go-shellwords v1.0.10 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/uuid v1.4.0 // indirect
github.com/imdario/mergo v0.3.14 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/klauspost/compress v1.17.4 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mattn/go-shellwords v1.0.12 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/sys/mountinfo v0.5.0 // indirect
github.com/moby/sys/signal v0.6.0 // indirect
github.com/moby/sys/mountinfo v0.6.2 // indirect
github.com/moby/sys/sequential v0.5.0 // indirect
github.com/moby/sys/signal v0.7.0 // indirect
github.com/moby/sys/user v0.3.0 // indirect
github.com/moby/sys/userns v0.1.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
github.com/opencontainers/runc v1.1.1 // indirect
github.com/opencontainers/selinux v1.10.1 // indirect
github.com/prometheus/client_golang v1.12.2 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.35.0 // indirect
github.com/prometheus/procfs v0.7.3 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/opencontainers/selinux v1.11.0 // indirect
github.com/prometheus/client_golang v1.19.0 // indirect
github.com/prometheus/client_model v0.6.0 // indirect
github.com/prometheus/common v0.51.1 // indirect
github.com/prometheus/procfs v0.13.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
go.opencensus.io v0.23.0 // indirect
golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
google.golang.org/grpc v1.43.0 // indirect
google.golang.org/protobuf v1.28.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect
go.opentelemetry.io/otel v1.21.0 // indirect
go.opentelemetry.io/otel/metric v1.21.0 // indirect
go.opentelemetry.io/otel/trace v1.21.0 // indirect
golang.org/x/net v0.38.0 // indirect
golang.org/x/sync v0.12.0 // indirect
golang.org/x/text v0.23.0 // indirect
google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect
google.golang.org/grpc v1.59.0 // indirect
google.golang.org/protobuf v1.35.2 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gotest.tools/v3 v3.0.3 // indirect
)

1270
go.sum

File diff suppressed because it is too large Load Diff

View File

@ -6,12 +6,8 @@
export ARCH="arm64"
if [ ! -d "/usr/local/go/bin" ]; then
echo "Downloading Go.."
curl -SLsf https://golang.org/dl/go1.16.6.linux-$ARCH.tar.gz --output /tmp/go.tgz
sudo rm -rf /usr/local/go/
sudo mkdir -p /usr/local/go/
sudo tar -xvf /tmp/go.tgz -C /usr/local/go/ --strip-components=1
curl -sLS https://get.arkade.dev | sudo sh
sudo -E arkade system install go
else
echo "Go already present, skipping."
fi
@ -29,7 +25,7 @@ git clone https://github.com/containerd/containerd
cd containerd
git fetch origin --tags
git checkout v1.6.4
git checkout v1.7.27
make
sudo make install

View File

@ -1,37 +0,0 @@
#!/bin/bash
# See pre-reqs:
# https://github.com/alexellis/containerd-arm
export ARCH="arm64"
if [ ! -d "/usr/local/go/bin" ]; then
echo "Downloading Go.."
curl -SLsf https://golang.org/dl/go1.16.6.linux-$ARCH.tar.gz --output /tmp/go.tgz
sudo rm -rf /usr/local/go/
sudo mkdir -p /usr/local/go/
sudo tar -xvf /tmp/go.tgz -C /usr/local/go/ --strip-components=1
else
echo "Go already present, skipping."
fi
export GOPATH=$HOME/go/
export PATH=$PATH:/usr/local/go/bin/
go version
echo "Building containerd"
mkdir -p $GOPATH/src/github.com/containerd
cd $GOPATH/src/github.com/containerd
git clone https://github.com/containerd/containerd
cd containerd
git fetch origin --tags
git checkout v1.6.4
make
sudo make install
sudo containerd --version

View File

@ -7,12 +7,8 @@
export ARCH="arm64"
if [ ! -d "/usr/local/go/bin" ]; then
echo "Downloading Go.."
curl -SLsf https://golang.org/dl/go1.16.6.linux-$ARCH.tar.gz --output /tmp/go.tgz
sudo rm -rf /usr/local/go/
sudo mkdir -p /usr/local/go/
sudo tar -xvf /tmp/go.tgz -C /usr/local/go/ --strip-components=1
curl -sLS https://get.arkade.dev | sudo sh
sudo -E arkade system install go
else
echo "Go already present, skipping."
fi
@ -30,7 +26,7 @@ git clone https://github.com/containerd/containerd
cd containerd
git fetch origin --tags
git checkout v1.6.4
git checkout v1.7.27
make
sudo make install

48
hack/enable-journal.sh Normal file
View File

@ -0,0 +1,48 @@
#!/bin/bash
# Copyright OpenFaaS Ltd 2025
# This script is for use with Droplets created on DigitalOcean, it will
# ensure that systemd-journald is configured to log to disk and that
# rsyslog is configured to read from systemd-journald.
# Without this change, no logs will be available in the journal, and only
# /var/log/syslog will be populated.
set -e
echo "Checking systemd-journald logs..."
JOURNAL_STATUS=$(journalctl --no-pager -n 10 2>&1)
if echo "$JOURNAL_STATUS" | grep -q "No journal files were found"; then
echo "No journal files found. Fixing logging configuration..."
else
echo "Journald appears to be logging. No changes needed."
exit 0
fi
# Backup original config before making changes
sudo cp /etc/systemd/journald.conf /etc/systemd/journald.conf.bak
# Ensure Storage is persistent
sudo sed -i '/^#Storage=/c\Storage=persistent' /etc/systemd/journald.conf
# Ensure logs are not forwarded only to syslog
sudo sed -i '/^#ForwardToSyslog=/c\ForwardToSyslog=no' /etc/systemd/journald.conf
# Restart systemd-journald
echo "Restarting systemd-journald..."
sudo systemctl restart systemd-journald
# Check if rsyslog already loads imjournal
if ! grep -q 'module(load="imjournal")' /etc/rsyslog.conf; then
echo "Adding imjournal module to rsyslog..."
echo 'module(load="imjournal" StateFile="/var/lib/rsyslog/imjournal.state")' | sudo tee -a /etc/rsyslog.conf
fi
# Restart rsyslog to apply changes
echo "Restarting rsyslog..."
sudo systemctl restart rsyslog
echo "Done. Checking if logs appear in journald..."
journalctl --no-pager -n 10

View File

@ -2,7 +2,7 @@
Description=faasd-provider
[Service]
MemoryLimit=500M
MemoryMax=500M
Environment="secret_mount_path={{.SecretMountPath}}"
Environment="basic_auth=true"
Environment="hosts_dir=/var/lib/faasd"

View File

@ -3,7 +3,7 @@ Description=faasd
After=faasd-provider.service
[Service]
MemoryLimit=500M
MemoryMax=500M
ExecStart=/usr/local/bin/faasd up
Restart=on-failure
RestartSec=10s

132
hack/install-edge.sh Normal file
View File

@ -0,0 +1,132 @@
#!/bin/bash
# Copyright OpenFaaS Ltd 2025
set -e # stop on error
set -o pipefail
export NEEDRESTART_MODE=a
export DEBIAN_FRONTEND=noninteractive
if [ "$EUID" -ne 0 ]; then
echo "Please run as root or with sudo"
exit
fi
has_dnf() {
[ -n "$(command -v dnf)" ]
}
has_apt_get() {
[ -n "$(command -v apt-get)" ]
}
has_pacman() {
[ -n "$(command -v pacman)" ]
}
install_required_packages() {
if $(has_apt_get); then
echo iptables-persistent iptables-persistent/autosave_v4 boolean false | sudo debconf-set-selections
echo iptables-persistent iptables-persistent/autosave_v6 boolean false | sudo debconf-set-selections
# Debian bullseye is missing iptables. Added to required packages
# to get it working in raspberry pi. No such known issues in
# other distros. Hence, adding only to this block.
# reference: https://github.com/openfaas/faasd/pull/237
apt-get update -yq
apt-get install -yq curl runc bridge-utils iptables iptables-persistent
elif $(has_dnf); then
dnf install -y \
--allowerasing \
--setopt=install_weak_deps=False \
curl runc iptables-services bridge-utils which
elif $(has_pacman); then
pacman -Syy
pacman -Sy curl runc bridge-utils
else
fatal "Could not find apt-get, yum, or pacman. Cannot install dependencies on this OS."
exit 1
fi
}
echo "OpenFaaS Edge combines faasd with OpenFaaS Standard"
echo ""
echo ""
echo "1. Installing required OS packages, set SKIP_OS=1 to skip this step"
echo ""
if [ -z "$SKIP_OS" ]; then
install_required_packages
fi
echo ""
echo "2. Downloading OCI image, and installing pre-requisites"
echo ""
if [ ! -x "$(command -v arkade)" ]; then
# For Centos, RHEL, Fedora, Amazon Linux, and Oracle Linux, use BINLOCATION=/usr/bin/
if $(has_dnf); then
BINLOCATION=/usr/bin/
fi
curl -sLS https://get.arkade.dev | BINLOCATION=${BINLOCATION} sh
fi
PATH=$PATH:$HOME/.arkade/bin
tmpdir=$(mktemp -d)
# Ensure all existing services are stopped when installing over an
# existing faasd installation
systemctl stop faasd || :
systemctl stop faasd-provider || :
systemctl stop containerd || :
killall -9 containerd-shim-runc-v2 || :
killall -9 faasd || :
# crane, or docker can also be used to download the OCI image and to extract it
# Rather than the :latest tag, a specific tag can be given
# Use "crane ls ghcr.io/openfaasltd/faasd-pro" to see available tags
${BINLOCATION}arkade oci install --path ${tmpdir} \
ghcr.io/openfaasltd/faasd-pro:latest
cd ${tmpdir}
./install.sh ./
if has_dnf; then
isRhelLike=true
else
isRhelLike=false
fi
binaryName="faasd"
if [ "$isRhelLike" = true ]; then
binaryName="/usr/local/bin/faasd"
fi
echo ""
echo "3.1 Commercial users can create their license key as follows:"
echo ""
echo "sudo mkdir -p /var/lib/faasd/secrets"
echo "sudo nano /var/lib/faasd/secrets/openfaas_license"
echo ""
echo "3.2 For personal, non-commercial use only, GitHub Sponsors of @openfaas (25USD+) can run:"
echo ""
echo "sudo -E ${binaryName} github login"
echo "sudo -E ${binaryName} activate"
echo ""
echo "4. Then perform the final installation steps"
echo ""
echo "sudo -E sh -c \"cd ${tmpdir}/var/lib/faasd && ${binaryName} install\""
echo ""
echo "5. Refer to the complete handbook and supplementary documentation at:"
echo ""
echo "http://store.openfaas.com/l/serverless-for-everyone-else?layout=profile"
echo ""
echo "https://docs.openfaas.com/edge/overview"
echo ""

View File

@ -36,13 +36,19 @@ if [ "$(id -u)" -eq 0 ]; then
fi
verify_system() {
arch=$(uname -m)
if [ "$arch" == "armv7l" ]; then
fatal 'faasd requires a 64-bit Operating System, see: https://github.com/openfaas/faasd/issues/364'
fi
if ! [ -d /run/systemd ]; then
fatal 'Can not find systemd to use as a process supervisor for faasd'
fi
}
has_yum() {
[ -n "$(command -v yum)" ]
has_dnf() {
[ -n "$(command -v dnf)" ]
}
has_apt_get() {
@ -61,14 +67,16 @@ install_required_packages() {
# reference: https://github.com/openfaas/faasd/pull/237
$SUDO apt-get update -y
$SUDO apt-get install -y curl runc bridge-utils iptables
elif $(has_yum); then
$SUDO yum check-update -y
$SUDO yum install -y curl runc iptables-services
elif $(has_dnf); then
$SUDO dnf install -y \
--allowerasing \
--setopt=install_weak_deps=False \
curl runc iptables-services bridge-utils
elif $(has_pacman); then
$SUDO pacman -Syy
$SUDO pacman -Sy curl runc bridge-utils
else
fatal "Could not find apt-get, yum, or pacman. Cannot install dependencies on this OS."
fatal "Could not find apt-get, dnf, or pacman. Cannot install dependencies on this OS."
exit 1
fi
}
@ -84,9 +92,13 @@ install_cni_plugins() {
}
install_containerd() {
CONTAINERD_VER=v1.6.4
CONTAINERD_VER=1.7.27
$SUDO systemctl unmask containerd || :
$SUDO $ARKADE system install containerd --systemd --version ${CONTAINERD_VER} --progress=false
arch=$(uname -m)
$SUDO $ARKADE system install containerd --systemd --version v${CONTAINERD_VER} --progress=false
sleep 5
}
@ -99,9 +111,6 @@ install_faasd() {
aarch64)
suffix=-arm64
;;
armv7l)
suffix=-armhf
;;
*)
echo "Unsupported architecture $arch"
exit 1
@ -125,7 +134,9 @@ install_caddy() {
if [ ! -z "${FAASD_DOMAIN}" ]; then
CADDY_VER=v2.4.3
arkade get --progress=false caddy -v ${CADDY_VER}
$SUDO install -m 755 $HOME/.arkade/bin/caddy /usr/local/bin/
# /usr/bin/caddy is specified in the upstream service file.
$SUDO install -m 755 $HOME/.arkade/bin/caddy /usr/bin/caddy
$SUDO curl -fSLs https://raw.githubusercontent.com/caddyserver/dist/master/init/caddy.service --output /etc/systemd/system/caddy.service

10
main.go
View File

@ -7,14 +7,6 @@ import (
"github.com/openfaas/faasd/cmd"
)
// These values will be injected into these variables at the build time.
var (
// GitCommit Git Commit SHA
GitCommit string
// Version version of the CLI
Version string
)
func main() {
if _, ok := os.LookupEnv("CONTAINER_ID"); ok {
@ -31,7 +23,7 @@ func main() {
os.Exit(0)
}
if err := cmd.Execute(Version, GitCommit); err != nil {
if err := cmd.Execute(); err != nil {
os.Exit(1)
}
return

View File

@ -5,7 +5,6 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"log"
"net"
"os"
@ -86,7 +85,7 @@ func InitNetwork() (gocni.CNI, error) {
}
netConfig := path.Join(CNIConfDir, defaultCNIConfFilename)
if err := ioutil.WriteFile(netConfig, []byte(defaultCNIConf), 644); err != nil {
if err := os.WriteFile(netConfig, []byte(defaultCNIConf), 0644); err != nil {
return nil, fmt.Errorf("cannot write network config: %s", defaultCNIConfFilename)
}
@ -151,7 +150,7 @@ func DeleteCNINetwork(ctx context.Context, cni gocni.CNI, client *containerd.Cli
func GetIPAddress(container string, PID uint32) (string, error) {
CNIDir := path.Join(CNIDataDir, defaultNetworkName)
files, err := ioutil.ReadDir(CNIDir)
files, err := os.ReadDir(CNIDir)
if err != nil {
return "", fmt.Errorf("failed to read CNI dir for container %s: %v", container, err)
}

View File

@ -1,7 +1,6 @@
package cninetwork
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
@ -15,7 +14,7 @@ eth1`
PID := uint32(621)
fullPath := filepath.Join(os.TempDir(), fileName)
err := ioutil.WriteFile(fullPath, []byte(body), 0700)
err := os.WriteFile(fullPath, []byte(body), 0700)
if err != nil {
t.Fatalf(err.Error())
}
@ -24,7 +23,6 @@ eth1`
}()
got, err := isCNIResultForPID(fullPath, container, PID)
if err != nil {
t.Fatalf(err.Error())
}
@ -43,7 +41,7 @@ eth1`
PID := uint32(621)
fullPath := filepath.Join(os.TempDir(), fileName)
err := ioutil.WriteFile(fullPath, []byte(body), 0700)
err := os.WriteFile(fullPath, []byte(body), 0700)
if err != nil {
t.Fatalf(err.Error())
}
@ -52,10 +50,10 @@ eth1`
}()
got, err := isCNIResultForPID(fullPath, container, PID)
if err != nil {
t.Fatalf(err.Error())
}
want := false
if got != want {
t.Fatalf("want %v, but got %v", want, got)

39
pkg/connectivity.go Normal file
View File

@ -0,0 +1,39 @@
package pkg
import (
"fmt"
"io"
"net/http"
"strings"
)
// ConnectivityCheck checks if the controller can reach the
// public Internet via HTTPS.
// A license is required to use OpenFaaS CE for Commercial Use.
func ConnectivityCheck() error {
req, err := http.NewRequest(http.MethodGet, "https://checkip.amazonaws.com", nil)
if err != nil {
return err
}
req.Header.Set("User-Agent", fmt.Sprintf("openfaas-ce/%s faas-netes", Version))
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
if req.Body != nil {
defer req.Body.Close()
}
if res.StatusCode != http.StatusOK {
var body []byte
if res.Body != nil {
body, _ = io.ReadAll(res.Body)
}
return fmt.Errorf("unexpected status code checking connectivity: %d, body: %s", res.StatusCode, strings.TrimSpace(string(body)))
}
return nil
}

View File

@ -1,7 +1,6 @@
package pkg
import (
"io/ioutil"
"log"
"os"
"strings"
@ -54,7 +53,7 @@ func (l *LocalResolver) rebuild() {
l.Mutex.Lock()
defer l.Mutex.Unlock()
fileData, fileErr := ioutil.ReadFile(l.Path)
fileData, fileErr := os.ReadFile(l.Path)
if fileErr != nil {
log.Printf("resolver rebuild error: %s", fileErr.Error())
return

View File

@ -105,12 +105,12 @@ func streamLogs(ctx context.Context, cmd *exec.Cmd, out io.ReadCloser, msgs chan
// will ensure `out` is closed and all related resources cleaned up
go func() {
err := cmd.Wait()
log.Println("wait result", err)
if err := cmd.Wait(); err != nil {
log.Printf("journalctl exited with error: %s", err)
}
}()
defer func() {
log.Println("closing journal stream")
close(msgs)
}()
@ -176,7 +176,6 @@ func parseEntry(entry map[string]string) (logs.Message, error) {
}
func logErrOut(out io.ReadCloser) {
defer log.Println("stderr closed")
defer out.Close()
io.Copy(log.Writer(), out)

View File

@ -20,9 +20,11 @@ func ReadFromEnv(hasEnv types.HasEnv) (*types.FaaSConfig, *ProviderConfig, error
serviceTimeout := types.ParseIntOrDurationValue(hasEnv.Getenv("service_timeout"), time.Second*60)
config.EnableHealth = true
config.ReadTimeout = serviceTimeout
config.WriteTimeout = serviceTimeout
config.EnableBasicAuth = true
config.MaxIdleConns = types.ParseIntValue(hasEnv.Getenv("max_idle_conns"), 1024)
config.MaxIdleConnsPerHost = types.ParseIntValue(hasEnv.Getenv("max_idle_conns_per_host"), 1024)
port := types.ParseIntValue(hasEnv.Getenv("port"), 8081)
config.TCPPort = &port

View File

@ -4,15 +4,16 @@ import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"log"
"net/http"
"github.com/containerd/containerd"
"github.com/containerd/containerd/namespaces"
gocni "github.com/containerd/go-cni"
"github.com/openfaas/faas/gateway/requests"
"github.com/openfaas/faas-provider/types"
"github.com/openfaas/faasd/pkg"
cninetwork "github.com/openfaas/faasd/pkg/cninetwork"
"github.com/openfaas/faasd/pkg/service"
)
@ -28,22 +29,24 @@ func MakeDeleteHandler(client *containerd.Client, cni gocni.CNI) func(w http.Res
defer r.Body.Close()
body, _ := ioutil.ReadAll(r.Body)
log.Printf("[Delete] request: %s\n", string(body))
body, _ := io.ReadAll(r.Body)
req := requests.DeleteFunctionRequest{}
err := json.Unmarshal(body, &req)
if err != nil {
log.Printf("[Delete] error parsing input: %s\n", err)
req := types.DeleteFunctionRequest{}
if err := json.Unmarshal(body, &req); err != nil {
log.Printf("[Delete] error parsing input: %s", err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
lookupNamespace := getRequestNamespace(readNamespaceFromQuery(r))
// namespace moved from the querystring into the body
namespace := req.Namespace
if namespace == "" {
namespace = pkg.DefaultFunctionNamespace
}
// Check if namespace exists, and it has the openfaas label
valid, err := validNamespace(client.NamespaceService(), lookupNamespace)
valid, err := validNamespace(client.NamespaceService(), namespace)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
@ -56,15 +59,15 @@ func MakeDeleteHandler(client *containerd.Client, cni gocni.CNI) func(w http.Res
name := req.FunctionName
function, err := GetFunction(client, name, lookupNamespace)
function, err := GetFunction(client, name, namespace)
if err != nil {
msg := fmt.Sprintf("service %s not found", name)
msg := fmt.Sprintf("function %s.%s not found", name, namespace)
log.Printf("[Delete] %s\n", msg)
http.Error(w, msg, http.StatusNotFound)
return
}
ctx := namespaces.WithNamespace(context.Background(), lookupNamespace)
ctx := namespaces.WithNamespace(context.Background(), namespace)
// TODO: this needs to still happen if the task is paused
if function.replicas != 0 {
@ -74,13 +77,12 @@ func MakeDeleteHandler(client *containerd.Client, cni gocni.CNI) func(w http.Res
}
}
containerErr := service.Remove(ctx, client, name)
if containerErr != nil {
log.Printf("[Delete] error removing %s, %s\n", name, containerErr)
http.Error(w, containerErr.Error(), http.StatusInternalServerError)
if err := service.Remove(ctx, client, name); err != nil {
log.Printf("[Delete] error removing %s, %s\n", name, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
log.Printf("[Delete] deleted %s\n", name)
log.Printf("[Delete] Removed: %s.%s\n", name, namespace)
}
}

View File

@ -4,7 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"log"
"net/http"
"os"
@ -17,7 +17,7 @@ import (
"github.com/containerd/containerd/namespaces"
"github.com/containerd/containerd/oci"
gocni "github.com/containerd/go-cni"
"github.com/docker/distribution/reference"
"github.com/distribution/reference"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/openfaas/faas-provider/types"
cninetwork "github.com/openfaas/faasd/pkg/cninetwork"
@ -39,13 +39,12 @@ func MakeDeployHandler(client *containerd.Client, cni gocni.CNI, secretMountPath
defer r.Body.Close()
body, _ := ioutil.ReadAll(r.Body)
log.Printf("[Deploy] request: %s\n", string(body))
body, _ := io.ReadAll(r.Body)
req := types.FunctionDeployment{}
err := json.Unmarshal(body, &req)
if err != nil {
log.Printf("[Deploy] - error parsing input: %s\n", err)
log.Printf("[Deploy] - error parsing input: %s", err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
@ -76,10 +75,15 @@ func MakeDeployHandler(client *containerd.Client, cni gocni.CNI, secretMountPath
name := req.Service
ctx := namespaces.WithNamespace(context.Background(), namespace)
deployErr := deploy(ctx, req, client, cni, namespaceSecretMountPath, alwaysPull)
if deployErr != nil {
log.Printf("[Deploy] error deploying %s, error: %s\n", name, deployErr)
http.Error(w, deployErr.Error(), http.StatusBadRequest)
if err := preDeploy(client, 1); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
log.Printf("[Deploy] error deploying %s, error: %s\n", name, err)
return
}
if err := deploy(ctx, req, client, cni, namespaceSecretMountPath, alwaysPull); err != nil {
log.Printf("[Deploy] error deploying %s, error: %s\n", name, err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}
@ -179,8 +183,27 @@ func deploy(ctx context.Context, req types.FunctionDeployment, client *container
}
// countFunctions returns the number of functions deployed along with a map with a count
// in each namespace
func countFunctions(client *containerd.Client) (int64, int64, error) {
count := int64(0)
namespaceCount := int64(0)
namespaces := ListNamespaces(client)
for _, namespace := range namespaces {
fns, err := ListFunctions(client, namespace)
if err != nil {
return 0, 0, err
}
namespaceCount++
count += int64(len(fns))
}
return count, namespaceCount, nil
}
func buildLabels(request *types.FunctionDeployment) (map[string]string, error) {
// Adapted from faas-swarm/handlers/deploy.go:buildLabels
labels := map[string]string{}
if request.Labels != nil {
@ -229,9 +252,8 @@ func createTask(ctx context.Context, container containerd.Container, cni gocni.C
log.Printf("%s has IP: %s.\n", name, ip)
_, waitErr := task.Wait(ctx)
if waitErr != nil {
return errors.Wrapf(waitErr, "Unable to wait for task to start: %s", name)
if _, err := task.Wait(ctx); err != nil {
return errors.Wrapf(err, "Unable to wait for task to start: %s", name)
}
if startErr := task.Start(ctx); startErr != nil {
@ -315,3 +337,17 @@ func withMemory(mem *specs.LinuxMemory) oci.SpecOpts {
return nil
}
}
func preDeploy(client *containerd.Client, additional int64) error {
count, countNs, err := countFunctions(client)
log.Printf("Function count: %d, Namespace count: %d\n", count, countNs)
if err != nil {
return err
} else if count+additional > faasdMaxFunctions {
return fmt.Errorf("the OpenFaaS CE EULA allows %d/%d function(s), upgrade to faasd Pro to continue", faasdMaxFunctions, count+additional)
} else if countNs > faasdMaxNs {
return fmt.Errorf("the OpenFaaS CE EULA allows %d/%d namespace(s), upgrade to faasd Pro to continue", faasdMaxNs, countNs)
}
return nil
}

View File

@ -2,71 +2,17 @@ package handlers
import (
"context"
"errors"
"fmt"
"log"
"strings"
"time"
"github.com/opencontainers/runtime-spec/specs-go"
"github.com/containerd/containerd"
"github.com/containerd/containerd/namespaces"
"github.com/openfaas/faasd/pkg"
faasd "github.com/openfaas/faasd/pkg"
"github.com/openfaas/faasd/pkg/cninetwork"
)
type Function struct {
name string
namespace string
image string
pid uint32
replicas int
IP string
labels map[string]string
annotations map[string]string
secrets []string
envVars map[string]string
envProcess string
memoryLimit int64
createdAt time.Time
}
// ListFunctions returns a map of all functions with running tasks on namespace
func ListFunctions(client *containerd.Client, namespace string) (map[string]*Function, error) {
// Check if namespace exists, and it has the openfaas label
valid, err := validNamespace(client.NamespaceService(), namespace)
if err != nil {
return nil, err
}
if !valid {
return nil, errors.New("namespace not valid")
}
ctx := namespaces.WithNamespace(context.Background(), namespace)
functions := make(map[string]*Function)
containers, err := client.Containers(ctx)
if err != nil {
return functions, err
}
for _, c := range containers {
name := c.ID()
f, err := GetFunction(client, name, namespace)
if err != nil {
log.Printf("error getting function %s: ", name)
return functions, err
}
functions[name] = &f
}
return functions, nil
}
// GetFunction returns a function that matches name
func GetFunction(client *containerd.Client, name string, namespace string) (Function, error) {
ctx := namespaces.WithNamespace(context.Background(), namespace)
@ -196,44 +142,6 @@ func buildLabelsAndAnnotations(ctrLabels map[string]string) (map[string]string,
return labels, annotations
}
func ListNamespaces(client *containerd.Client) []string {
set := []string{}
store := client.NamespaceService()
namespaces, err := store.List(context.Background())
if err != nil {
log.Printf("Error listing namespaces: %s", err.Error())
set = append(set, faasd.DefaultFunctionNamespace)
return set
}
for _, namespace := range namespaces {
labels, err := store.Labels(context.Background(), namespace)
if err != nil {
log.Printf("Error listing label for namespace %s: %s", namespace, err.Error())
continue
}
if _, found := labels[pkg.NamespaceLabel]; found {
set = append(set, namespace)
}
if !findNamespace(faasd.DefaultFunctionNamespace, set) {
set = append(set, faasd.DefaultFunctionNamespace)
}
}
return set
}
func findNamespace(target string, items []string) bool {
for _, n := range items {
if n == target {
return true
}
}
return false
}
func readMemoryLimitFromSpec(spec *specs.Spec) int64 {
if spec.Linux == nil || spec.Linux.Resources == nil || spec.Linux.Resources.Memory == nil || spec.Linux.Resources.Memory.Limit == nil {
return 0

View File

@ -0,0 +1,65 @@
package handlers
import (
"context"
"errors"
"log"
"strings"
"time"
"github.com/containerd/containerd"
"github.com/containerd/containerd/namespaces"
)
type Function struct {
name string
namespace string
image string
pid uint32
replicas int
IP string
labels map[string]string
annotations map[string]string
secrets []string
envVars map[string]string
envProcess string
memoryLimit int64
createdAt time.Time
}
// ListFunctions returns a map of all functions with running tasks on namespace
func ListFunctions(client *containerd.Client, namespace string) (map[string]*Function, error) {
// Check if namespace exists, and it has the openfaas label
valid, err := validNamespace(client.NamespaceService(), namespace)
if err != nil {
return nil, err
}
if !valid {
return nil, errors.New("namespace not valid")
}
ctx := namespaces.WithNamespace(context.Background(), namespace)
functions := make(map[string]*Function)
containers, err := client.Containers(ctx)
if err != nil {
return functions, err
}
for _, c := range containers {
name := c.ID()
f, err := GetFunction(client, name, namespace)
if err != nil {
if !strings.Contains(err.Error(), "unable to get IP address for container") {
log.Printf("List functions, skipping: %s, error: %s", name, err)
}
} else {
functions[name] = &f
}
}
return functions, nil
}

View File

@ -12,10 +12,10 @@ const (
OrchestrationIdentifier = "containerd"
// ProviderName name of the provider
ProviderName = "faasd"
ProviderName = "faasd-ce"
)
//MakeInfoHandler creates handler for /system/info endpoint
// MakeInfoHandler creates handler for /system/info endpoint
func MakeInfoHandler(version, sha string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if r.Body != nil {
@ -31,8 +31,8 @@ func MakeInfoHandler(version, sha string) http.HandlerFunc {
},
}
jsonOut, marshalErr := json.Marshal(infoResponse)
if marshalErr != nil {
jsonOut, err := json.Marshal(infoResponse)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return
}
@ -42,3 +42,6 @@ func MakeInfoHandler(version, sha string) http.HandlerFunc {
w.Write(jsonOut)
}
}
const faasdMaxFunctions = 15
const faasdMaxNs = 1

View File

@ -24,7 +24,7 @@ func (i *InvokeResolver) Resolve(functionName string) (url.URL, error) {
actualFunctionName := functionName
log.Printf("Resolve: %q\n", actualFunctionName)
namespace := getNamespace(functionName, faasd.DefaultFunctionNamespace)
namespace := getNamespaceOrDefault(functionName, faasd.DefaultFunctionNamespace)
if strings.Contains(functionName, ".") {
actualFunctionName = strings.TrimSuffix(functionName, "."+namespace)
@ -47,7 +47,7 @@ func (i *InvokeResolver) Resolve(functionName string) (url.URL, error) {
return *urlRes, nil
}
func getNamespace(name, defaultNamespace string) string {
func getNamespaceOrDefault(name, defaultNamespace string) string {
namespace := defaultNamespace
if strings.Contains(name, ".") {
namespace = name[strings.LastIndexAny(name, ".")+1:]

View File

@ -0,0 +1,285 @@
package handlers
import (
"context"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"strings"
"github.com/containerd/containerd"
"github.com/gorilla/mux"
"github.com/openfaas/faas-provider/types"
)
func MakeMutateNamespace(client *containerd.Client) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
if r.Body != nil {
defer r.Body.Close()
}
switch r.Method {
case http.MethodPost:
createNamespace(client, w, r)
case http.MethodGet:
getNamespace(client, w, r)
case http.MethodDelete:
deleteNamespace(client, w, r)
case http.MethodPut:
updateNamespace(client, w, r)
default:
w.WriteHeader(http.StatusMethodNotAllowed)
}
}
}
func updateNamespace(client *containerd.Client, w http.ResponseWriter, r *http.Request) {
req, err := parseNamespaceRequest(r)
if err != nil {
http.Error(w, err.Error(), err.(*HttpError).Status)
return
}
namespaceExists, err := namespaceExists(r.Context(), client, req.Name)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if !namespaceExists {
http.Error(w, fmt.Sprintf("namespace %s not found", req.Name), http.StatusNotFound)
return
}
originalLabels, err := client.NamespaceService().Labels(r.Context(), req.Name)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if !hasOpenFaaSLabel(originalLabels) {
http.Error(w, fmt.Sprintf("namespace %s is not an openfaas namespace", req.Name), http.StatusBadRequest)
return
}
var exclusions []string
// build exclusions
for key, _ := range originalLabels {
if _, ok := req.Labels[key]; !ok {
exclusions = append(exclusions, key)
}
}
// Call SetLabel with empty string if label is to be removed
for _, key := range exclusions {
if err := client.NamespaceService().SetLabel(r.Context(), req.Name, key, ""); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
// Now add the new labels
for key, value := range req.Labels {
if err := client.NamespaceService().SetLabel(r.Context(), req.Name, key, value); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
w.WriteHeader(http.StatusAccepted)
}
func deleteNamespace(client *containerd.Client, w http.ResponseWriter, r *http.Request) {
req, err := parseNamespaceRequest(r)
if err != nil {
http.Error(w, err.Error(), err.(*HttpError).Status)
return
}
if err := client.NamespaceService().Delete(r.Context(), req.Name); err != nil {
if strings.Contains(err.Error(), "not found") {
http.Error(w, fmt.Sprintf("namespace %s not found", req.Name), http.StatusNotFound)
return
}
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusAccepted)
}
func namespaceExists(ctx context.Context, client *containerd.Client, name string) (bool, error) {
ns, err := client.NamespaceService().List(ctx)
if err != nil {
return false, err
}
found := false
for _, namespace := range ns {
if namespace == name {
found = true
break
}
}
return found, nil
}
func getNamespace(client *containerd.Client, w http.ResponseWriter, r *http.Request) {
req, err := parseNamespaceRequest(r)
if err != nil {
http.Error(w, err.Error(), err.(*HttpError).Status)
return
}
namespaceExists, err := namespaceExists(r.Context(), client, req.Name)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if !namespaceExists {
http.Error(w, fmt.Sprintf("namespace %s not found", req.Name), http.StatusNotFound)
return
}
labels, err := client.NamespaceService().Labels(r.Context(), req.Name)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if !hasOpenFaaSLabel(labels) {
http.Error(w, fmt.Sprintf("namespace %s not found", req.Name), http.StatusNotFound)
return
}
res := types.FunctionNamespace{
Name: req.Name,
Labels: labels,
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(res); err != nil {
log.Printf("Get Namespace error: %s", err)
}
}
func createNamespace(client *containerd.Client, w http.ResponseWriter, r *http.Request) {
req, err := parseNamespaceRequest(r)
if err != nil {
http.Error(w, err.Error(), err.(*HttpError).Status)
return
}
// Check if namespace exists, and it has the openfaas label
namespaces, err := client.NamespaceService().List(r.Context())
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
found := false
for _, namespace := range namespaces {
if namespace == req.Name {
found = true
break
}
}
if found {
http.Error(w, fmt.Sprintf("namespace %s already exists", req.Name), http.StatusConflict)
return
}
if err := client.NamespaceService().Create(r.Context(), req.Name, req.Labels); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
w.WriteHeader(http.StatusCreated)
}
// getNamespace returns a namespace object or an error
func parseNamespaceRequest(r *http.Request) (types.FunctionNamespace, error) {
var req types.FunctionNamespace
vars := mux.Vars(r)
namespaceInPath := vars["name"]
if r.Method == http.MethodGet {
if namespaceInPath == "" {
return req, &HttpError{
Err: fmt.Errorf("namespace not specified in URL"),
Status: http.StatusBadRequest,
}
}
return types.FunctionNamespace{
Name: namespaceInPath,
}, nil
}
body, _ := io.ReadAll(r.Body)
if err := json.Unmarshal(body, &req); err != nil {
return req, &HttpError{
Err: fmt.Errorf("error parsing request body: %s", err.Error()),
Status: http.StatusBadRequest,
}
}
if r.Method != http.MethodPost {
if namespaceInPath == "" {
return req, &HttpError{
Err: fmt.Errorf("namespace not specified in URL"),
Status: http.StatusBadRequest,
}
}
if req.Name != namespaceInPath {
return req, &HttpError{
Err: fmt.Errorf("namespace in request body does not match namespace in URL"),
Status: http.StatusBadRequest,
}
}
}
if req.Name == "" {
return req, &HttpError{
Err: fmt.Errorf("namespace not specified in request body"),
Status: http.StatusBadRequest,
}
}
if ok := hasOpenFaaSLabel(req.Labels); !ok {
return req, &HttpError{
Err: fmt.Errorf("request does not have openfaas=1 label"),
Status: http.StatusBadRequest,
}
}
return req, nil
}
func hasOpenFaaSLabel(labels map[string]string) bool {
if v, ok := labels["openfaas"]; ok && v == "1" {
return true
}
return false
}
type HttpError struct {
Err error
Status int
}
func (e *HttpError) Error() string {
return e.Err.Error()
}

View File

@ -1,10 +1,14 @@
package handlers
import (
"context"
"encoding/json"
"log"
"net/http"
"github.com/containerd/containerd"
"github.com/openfaas/faasd/pkg"
faasd "github.com/openfaas/faasd/pkg"
)
func MakeNamespacesLister(client *containerd.Client) func(w http.ResponseWriter, r *http.Request) {
@ -16,3 +20,43 @@ func MakeNamespacesLister(client *containerd.Client) func(w http.ResponseWriter,
w.Write(body)
}
}
func ListNamespaces(client *containerd.Client) []string {
set := []string{faasd.DefaultFunctionNamespace}
store := client.NamespaceService()
namespaces, err := store.List(context.Background())
if err != nil {
log.Printf("Error listing namespaces: %s", err.Error())
return set
}
for _, namespace := range namespaces {
labels, err := store.Labels(context.Background(), namespace)
if err != nil {
log.Printf("Error listing label for namespace %s: %s", namespace, err.Error())
continue
}
if _, found := labels[pkg.NamespaceLabel]; found {
set = append(set, namespace)
}
}
if len(set) == 0 {
set = append(set, faasd.DefaultFunctionNamespace)
}
return set
}
func findNamespace(target string, items []string) bool {
for _, n := range items {
if n == target {
return true
}
}
return false
}

View File

@ -31,7 +31,7 @@ func MakeReadHandler(client *containerd.Client) func(w http.ResponseWriter, r *h
res := []types.FunctionStatus{}
fns, err := ListFunctions(client, lookupNamespace)
if err != nil {
log.Printf("[Read] error listing functions. Error: %s\n", err)
log.Printf("[Read] error listing functions. Error: %s", err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}

View File

@ -4,7 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"log"
"net/http"
@ -13,6 +13,7 @@ import (
gocni "github.com/containerd/go-cni"
"github.com/openfaas/faas-provider/types"
"github.com/openfaas/faasd/pkg"
)
func MakeReplicaUpdateHandler(client *containerd.Client, cni gocni.CNI) func(w http.ResponseWriter, r *http.Request) {
@ -26,20 +27,20 @@ func MakeReplicaUpdateHandler(client *containerd.Client, cni gocni.CNI) func(w h
defer r.Body.Close()
body, _ := ioutil.ReadAll(r.Body)
log.Printf("[Scale] request: %s\n", string(body))
body, _ := io.ReadAll(r.Body)
req := types.ScaleServiceRequest{}
err := json.Unmarshal(body, &req)
if err != nil {
log.Printf("[Scale] error parsing input: %s\n", err)
if err := json.Unmarshal(body, &req); err != nil {
log.Printf("[Scale] error parsing input: %s", err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
namespace := getRequestNamespace(readNamespaceFromQuery(r))
namespace := req.Namespace
if namespace == "" {
namespace = pkg.DefaultFunctionNamespace
}
// Check if namespace exists, and it has the openfaas label
valid, err := validNamespace(client.NamespaceService(), namespace)
@ -56,7 +57,7 @@ func MakeReplicaUpdateHandler(client *containerd.Client, cni gocni.CNI) func(w h
name := req.ServiceName
if _, err := GetFunction(client, name, namespace); err != nil {
msg := fmt.Sprintf("service %s not found", name)
msg := fmt.Sprintf("function: %s.%s not found", name, namespace)
log.Printf("[Scale] %s\n", msg)
http.Error(w, msg, http.StatusNotFound)
return
@ -95,32 +96,24 @@ func MakeReplicaUpdateHandler(client *containerd.Client, cni gocni.CNI) func(w h
createNewTask := false
// Scale to zero
if req.Replicas == 0 {
// If a task is running, pause it
if taskExists && taskStatus.Status == containerd.Running {
if pauseErr := task.Pause(ctx); pauseErr != nil {
wrappedPauseErr := fmt.Errorf("error pausing task %s, error: %s", name, pauseErr)
log.Printf("[Scale] %s\n", wrappedPauseErr.Error())
http.Error(w, wrappedPauseErr.Error(), http.StatusNotFound)
http.Error(w, "replicas must > 0 for faasd CE", http.StatusBadRequest)
return
}
}
}
if taskExists {
if taskStatus != nil {
if taskStatus.Status == containerd.Paused {
if resumeErr := task.Resume(ctx); resumeErr != nil {
log.Printf("[Scale] error resuming task %s, error: %s\n", name, resumeErr)
http.Error(w, resumeErr.Error(), http.StatusBadRequest)
if _, err := task.Delete(ctx); err != nil {
log.Printf("[Scale] error deleting paused task %s, error: %s\n", name, err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
} else if taskStatus.Status == containerd.Stopped {
// Stopped tasks cannot be restarted, must be removed, and created again
if _, delErr := task.Delete(ctx); delErr != nil {
log.Printf("[Scale] error deleting stopped task %s, error: %s\n", name, delErr)
http.Error(w, delErr.Error(), http.StatusBadRequest)
if _, err := task.Delete(ctx); err != nil {
log.Printf("[Scale] error deleting stopped task %s, error: %s\n", name, err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
createNewTask = true

View File

@ -3,7 +3,7 @@ package handlers
import (
"encoding/json"
"fmt"
"io/ioutil"
"io"
"log"
"net/http"
"os"
@ -71,7 +71,7 @@ func listSecrets(store provider.Labeller, w http.ResponseWriter, r *http.Request
}
if err != nil {
fmt.Printf("Error Occured: %s \n", err)
log.Printf("[Secret] Error listing secrets: %s ", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
@ -116,7 +116,7 @@ func createSecret(w http.ResponseWriter, r *http.Request, mountPath string) {
data = []byte(secret.Value)
}
err = ioutil.WriteFile(path.Join(mountPath, secret.Name), data, secretFilePermission)
err = os.WriteFile(path.Join(mountPath, secret.Name), data, secretFilePermission)
if err != nil {
log.Printf("[secret] error %s", err.Error())
@ -147,7 +147,7 @@ func deleteSecret(w http.ResponseWriter, r *http.Request, mountPath string) {
func parseSecret(r *http.Request) (types.Secret, error) {
secret := types.Secret{}
bytesOut, err := ioutil.ReadAll(r.Body)
bytesOut, err := io.ReadAll(r.Body)
if err != nil {
return secret, err
}

View File

@ -3,7 +3,7 @@ package handlers
import (
"encoding/json"
"fmt"
"io/ioutil"
"io"
"net/http"
"net/http/httptest"
"os"
@ -232,7 +232,7 @@ func TestListSecrets(t *testing.T) {
t.Fatalf("want error message: %q, but got %q", tc.err, w.Body.String())
}
body, err := ioutil.ReadAll(resp.Body)
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatalf("can't read response of list %v", err)
}

View File

@ -4,7 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"io"
"log"
"net/http"
@ -28,17 +28,17 @@ func MakeUpdateHandler(client *containerd.Client, cni gocni.CNI, secretMountPath
defer r.Body.Close()
body, _ := ioutil.ReadAll(r.Body)
log.Printf("[Update] request: %s\n", string(body))
body, _ := io.ReadAll(r.Body)
req := types.FunctionDeployment{}
err := json.Unmarshal(body, &req)
if err != nil {
log.Printf("[Update] error parsing input: %s\n", err)
log.Printf("[Update] error parsing input: %s", err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
name := req.Service
namespace := getRequestNamespace(req.Namespace)
@ -54,11 +54,17 @@ func MakeUpdateHandler(client *containerd.Client, cni gocni.CNI, secretMountPath
return
}
if err := preDeploy(client, int64(0)); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
log.Printf("[Deploy] error deploying %s, error: %s\n", name, err)
return
}
namespaceSecretMountPath := getNamespaceSecretMountPath(secretMountPath, namespace)
function, err := GetFunction(client, name, namespace)
if err != nil {
msg := fmt.Sprintf("service %s not found", name)
msg := fmt.Sprintf("function: %s.%s not found", name, namespace)
log.Printf("[Update] %s\n", msg)
http.Error(w, msg, http.StatusNotFound)
return

View File

@ -39,7 +39,8 @@ func validNamespace(store provider.Labeller, namespace string) (bool, error) {
return false, err
}
if value, found := labels[pkg.NamespaceLabel]; found && value == "true" {
// check for true to keep it backward compatible
if value, found := labels[pkg.NamespaceLabel]; found && (value == "true" || value == "1") {
return true, nil
}

View File

@ -75,18 +75,18 @@ func (p *Proxy) Start() error {
// Wait for a connection.
conn, err := l.Accept()
if err != nil {
acceptErr := fmt.Errorf("Unable to accept on %d, error: %s",
log.Printf("Unable to accept on: %d, error: %s",
p.Port,
err.Error())
log.Printf("%s", acceptErr.Error())
return acceptErr
return err
}
upstream, err := net.Dial("tcp", upstreamAddr)
if err != nil {
log.Printf("unable to dial to %s, error: %s", upstreamAddr, err.Error())
return err
conn.Close()
log.Printf("Unable to dial: %s, error: %s", upstreamAddr, err.Error())
continue
}
go pipe(conn, upstream)

View File

@ -2,7 +2,7 @@ package pkg
import (
"fmt"
"io/ioutil"
"io"
"log"
"net/http"
"net/http/httptest"
@ -61,7 +61,7 @@ func Test_Proxy_ToPrivateServer(t *testing.T) {
time.Sleep(time.Millisecond * 100)
} else {
resBody, _ := ioutil.ReadAll(res.Body)
resBody, _ := io.ReadAll(res.Body)
if string(resBody) != string(wantBody) {
t.Errorf("want %s, but got %s in body", string(wantBody), string(resBody))
}

View File

@ -6,6 +6,7 @@ import (
"log"
"os"
"path/filepath"
"strings"
"sync"
"time"
@ -45,10 +46,24 @@ func Remove(ctx context.Context, client *containerd.Client, name string) error {
log.Printf("Status of %s is: %s\n", name, status.Status)
}
log.Printf("Need to kill task: %s\n", name)
if err = killTask(ctx, t); err != nil {
var gracePeriod = time.Second * 30
spec, err := t.Spec(ctx)
if err == nil {
for _, p := range spec.Process.Env {
k, v, ok := strings.Cut(p, "=")
if ok && k == "grace_period" {
periodVal, err := time.ParseDuration(v)
if err == nil {
gracePeriod = periodVal
}
}
}
}
if err = killTask(ctx, t, gracePeriod); err != nil {
return fmt.Errorf("error killing task %s, %s, %w", container.ID(), name, err)
}
}
if err := container.Delete(ctx, containerd.WithSnapshotCleanup); err != nil {
@ -57,7 +72,7 @@ func Remove(ctx context.Context, client *containerd.Client, name string) error {
} else {
service := client.SnapshotService("")
key := name + "snapshot"
key := name + "-snapshot"
if _, err := client.SnapshotService("").Stat(ctx, key); err == nil {
service.Remove(ctx, key)
}
@ -66,14 +81,13 @@ func Remove(ctx context.Context, client *containerd.Client, name string) error {
}
// Adapted from Stellar - https://github.com/stellar
func killTask(ctx context.Context, task containerd.Task) error {
killTimeout := 30 * time.Second
func killTask(ctx context.Context, task containerd.Task, gracePeriod time.Duration) error {
wg := &sync.WaitGroup{}
wg.Add(1)
var err error
waited := false
go func() {
defer wg.Done()
if task != nil {
@ -89,22 +103,39 @@ func killTask(ctx context.Context, task containerd.Task) error {
select {
case <-wait:
task.Delete(ctx)
waited = true
return
case <-time.After(killTimeout):
case <-time.After(gracePeriod):
log.Printf("Sending SIGKILL to: %s after: %s", task.ID(), gracePeriod.Round(time.Second).String())
if err := task.Kill(ctx, unix.SIGKILL, containerd.WithKillAll); err != nil {
log.Printf("error force killing container task: %s", err)
log.Printf("error sending SIGKILL to task: %s", err)
}
return
}
}
}()
wg.Wait()
if task != nil {
if !waited {
wait, err := task.Wait(ctx)
if err != nil {
log.Printf("error waiting on task after kill: %s", err)
}
<-wait
}
if _, err := task.Delete(ctx); err != nil {
return err
}
}
return err
}
func getResolver(ctx context.Context, configFile *configfile.ConfigFile) (remotes.Resolver, error) {
func getResolver(configFile *configfile.ConfigFile) (remotes.Resolver, error) {
// credsFunc is based on https://github.com/moby/buildkit/blob/0b130cca040246d2ddf55117eeff34f546417e40/session/auth/authprovider/authprovider.go#L35
credFunc := func(host string) (string, string, error) {
if host == "registry-1.docker.io" {
@ -139,7 +170,7 @@ func PrepareImage(ctx context.Context, client *containerd.Client, imageName, sna
if err != nil {
return nil, err
}
resolver, err = getResolver(ctx, configFile)
resolver, err = getResolver(configFile)
if err != nil {
return empty, err
}

View File

@ -3,7 +3,6 @@ package pkg
import (
"context"
"fmt"
"io/ioutil"
"log"
"os"
"path"
@ -11,7 +10,7 @@ import (
"strconv"
"strings"
"github.com/alexellis/k3sup/pkg/env"
"github.com/alexellis/arkade/pkg/env"
"github.com/compose-spec/compose-go/loader"
compose "github.com/compose-spec/compose-go/types"
"github.com/containerd/containerd"
@ -19,7 +18,7 @@ import (
"github.com/containerd/containerd/containers"
"github.com/containerd/containerd/oci"
gocni "github.com/containerd/go-cni"
"github.com/docker/distribution/reference"
"github.com/distribution/reference"
"github.com/openfaas/faasd/pkg/cninetwork"
"github.com/openfaas/faasd/pkg/service"
"github.com/pkg/errors"
@ -57,8 +56,14 @@ type ServicePort struct {
}
type Mount struct {
// Src relative to the working directory for faasd
Src string
// Dest is the absolute path within the container
Dest string
// ReadOnly when set to true indicates the mount will be set to "ro" instead of "rw"
ReadOnly bool
}
type Supervisor struct {
@ -96,7 +101,7 @@ func (s *Supervisor) Start(svcs []Service) error {
127.0.0.1 localhost
%s faasd-provider`, gw)
writeHostsErr := ioutil.WriteFile(path.Join(wd, "hosts"),
writeHostsErr := os.WriteFile(path.Join(wd, "hosts"),
[]byte(hosts), workingDirectoryPermission)
if writeHostsErr != nil {
@ -151,11 +156,18 @@ func (s *Supervisor) Start(svcs []Service) error {
mounts := []specs.Mount{}
if len(svc.Mounts) > 0 {
for _, mnt := range svc.Mounts {
var options = []string{"rbind"}
if mnt.ReadOnly {
options = append(options, "ro")
} else {
options = append(options, "rw")
}
mounts = append(mounts, specs.Mount{
Source: mnt.Src,
Destination: mnt.Dest,
Type: "bind",
Options: []string{"rbind", "rw"},
Options: options,
})
// Only create directories, not files.
@ -215,7 +227,7 @@ func (s *Supervisor) Start(svcs []Service) error {
)
if err != nil {
log.Printf("Error creating container: %s\n", err)
log.Printf("Error creating container: %s", err)
return err
}
@ -223,7 +235,7 @@ func (s *Supervisor) Start(svcs []Service) error {
task, err := newContainer.NewTask(ctx, cio.BinaryIO("/usr/local/bin/faasd", nil))
if err != nil {
log.Printf("Error creating task: %s\n", err)
log.Printf("Error creating task: %s", err)
return err
}
@ -242,7 +254,7 @@ func (s *Supervisor) Start(svcs []Service) error {
log.Printf("%s has IP: %s\n", newContainer.ID(), ip)
hosts, err := ioutil.ReadFile("hosts")
hosts, err := os.ReadFile("hosts")
if err != nil {
log.Printf("Unable to read hosts file: %s\n", err.Error())
}
@ -251,12 +263,12 @@ func (s *Supervisor) Start(svcs []Service) error {
%s %s
`, ip, svc.Name))
if err := ioutil.WriteFile("hosts", hosts, workingDirectoryPermission); err != nil {
if err := os.WriteFile("hosts", hosts, workingDirectoryPermission); err != nil {
log.Printf("Error writing file: %s %s\n", "hosts", err)
}
if _, err := task.Wait(ctx); err != nil {
log.Printf("Task wait error: %s\n", err)
log.Printf("Task wait error: %s", err)
return err
}
@ -264,7 +276,7 @@ func (s *Supervisor) Start(svcs []Service) error {
// log.Println("Exited: ", exitStatusC)
if err = task.Start(ctx); err != nil {
log.Printf("Task start error: %s\n", err)
log.Printf("Task start error: %s", err)
return err
}
}
@ -344,6 +356,7 @@ func ParseCompose(config *compose.Config) ([]Service, error) {
mounts = append(mounts, Mount{
Src: v.Source,
Dest: v.Target,
ReadOnly: v.ReadOnly,
})
}
@ -386,7 +399,7 @@ func LoadComposeFile(wd string, file string) (*compose.Config, error) {
func LoadComposeFileWithArch(wd string, file string, archGetter ArchGetter) (*compose.Config, error) {
file = path.Join(wd, file)
b, err := ioutil.ReadFile(file)
b, err := os.ReadFile(file)
if err != nil {
return nil, err
}
@ -438,8 +451,6 @@ func GetArchSuffix(getClientArch ArchGetter) (suffix string, err error) {
case "x86_64":
// no suffix needed
return "", nil
case "armhf", "armv7l":
return "-armhf", nil
case "arm64", "aarch64":
return "-arm64", nil
default:

View File

@ -180,7 +180,7 @@ func equalMountSlice(t *testing.T, want, found []Mount) {
for i := range want {
if !reflect.DeepEqual(want[i], found[i]) {
t.Fatalf("unexpected value at postition %d: want %s, got %s", i, want[i], found[i])
t.Fatalf("unexpected value at postition %d: want %v, got %v", i, want[i], found[i])
}
}
}
@ -210,18 +210,6 @@ func Test_GetArchSuffix(t *testing.T) {
foundArch: "anything_else",
want: "",
},
{
name: "armhf has armhf suffix",
foundOS: "Linux",
foundArch: "armhf",
want: "-armhf",
},
{
name: "armv7l has armhf suffix",
foundOS: "Linux",
foundArch: "armv7l",
want: "-armhf",
},
{
name: "arm64 has arm64 suffix",
foundOS: "Linux",

View File

@ -2,21 +2,23 @@ package systemd
import (
"bytes"
"context"
"fmt"
"os"
"path/filepath"
"text/template"
execute "github.com/alexellis/go-execute/pkg/v1"
execute "github.com/alexellis/go-execute/v2"
)
func Enable(unit string) error {
task := execute.ExecTask{Command: "systemctl",
task := execute.ExecTask{
Command: "systemctl",
Args: []string{"enable", unit},
StreamStdio: false,
}
res, err := task.Execute()
res, err := task.Execute(context.Background())
if err != nil {
return err
}
@ -29,12 +31,13 @@ func Enable(unit string) error {
}
func Start(unit string) error {
task := execute.ExecTask{Command: "systemctl",
task := execute.ExecTask{
Command: "systemctl",
Args: []string{"start", unit},
StreamStdio: false,
}
res, err := task.Execute()
res, err := task.Execute(context.Background())
if err != nil {
return err
}
@ -47,12 +50,13 @@ func Start(unit string) error {
}
func DaemonReload() error {
task := execute.ExecTask{Command: "systemctl",
task := execute.ExecTask{
Command: "systemctl",
Args: []string{"daemon-reload"},
StreamStdio: false,
}
res, err := task.Execute()
res, err := task.Execute(context.Background())
if err != nil {
return err
}
@ -71,23 +75,20 @@ func InstallUnit(name string, tokens map[string]string) error {
tmplName := "./hack/" + name + ".service"
tmpl, err := template.ParseFiles(tmplName)
if err != nil {
return fmt.Errorf("error loading template %s, error %s", tmplName, err)
}
var tpl bytes.Buffer
err = tmpl.Execute(&tpl, tokens)
if err != nil {
if err := tmpl.Execute(&tpl, tokens); err != nil {
return err
}
err = writeUnit(name+".service", tpl.Bytes())
if err != nil {
if err := writeUnit(name+".service", tpl.Bytes()); err != nil {
return err
}
return nil
}
@ -96,7 +97,12 @@ func writeUnit(name string, data []byte) error {
if err != nil {
return err
}
defer f.Close()
_, err = f.Write(data)
if _, err := f.Write(data); err != nil {
return err
}
return nil
}

View File

@ -1 +1,17 @@
package pkg
// These values will be injected into these variables at the build time.
var (
// GitCommit Git Commit SHA
GitCommit string
// Version version of the CLI
Version string
)
// GetVersion get latest version
func GetVersion() string {
if len(Version) == 0 {
return "dev"
}
return Version
}

View File

@ -1,4 +1,4 @@
Apache License
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
@ -178,7 +178,7 @@ Apache License
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
@ -186,8 +186,7 @@ Apache License
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2015, Google Inc
Copyright 2018, GoGo Authors
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -200,4 +199,3 @@ Apache License
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

93
vendor/github.com/AdaLogics/go-fuzz-headers/README.md generated vendored Normal file
View File

@ -0,0 +1,93 @@
# go-fuzz-headers
This repository contains various helper functions for go fuzzing. It is mostly used in combination with [go-fuzz](https://github.com/dvyukov/go-fuzz), but compatibility with fuzzing in the standard library will also be supported. Any coverage guided fuzzing engine that provides an array or slice of bytes can be used with go-fuzz-headers.
## Usage
Using go-fuzz-headers is easy. First create a new consumer with the bytes provided by the fuzzing engine:
```go
import (
fuzz "github.com/AdaLogics/go-fuzz-headers"
)
data := []byte{'R', 'a', 'n', 'd', 'o', 'm'}
f := fuzz.NewConsumer(data)
```
This creates a `Consumer` that consumes the bytes of the input as it uses them to fuzz different types.
After that, `f` can be used to easily create fuzzed instances of different types. Below are some examples:
### Structs
One of the most useful features of go-fuzz-headers is its ability to fill structs with the data provided by the fuzzing engine. This is done with a single line:
```go
type Person struct {
Name string
Age int
}
p := Person{}
// Fill p with values based on the data provided by the fuzzing engine:
err := f.GenerateStruct(&p)
```
This includes nested structs too. In this example, the fuzz Consumer will also insert values in `p.BestFriend`:
```go
type PersonI struct {
Name string
Age int
BestFriend PersonII
}
type PersonII struct {
Name string
Age int
}
p := PersonI{}
err := f.GenerateStruct(&p)
```
If the consumer should insert values for unexported fields as well as exported, this can be enabled with:
```go
f.AllowUnexportedFields()
```
...and disabled with:
```go
f.DisallowUnexportedFields()
```
### Other types:
Other useful APIs:
```go
createdString, err := f.GetString() // Gets a string
createdInt, err := f.GetInt() // Gets an integer
createdByte, err := f.GetByte() // Gets a byte
createdBytes, err := f.GetBytes() // Gets a byte slice
createdBool, err := f.GetBool() // Gets a boolean
err := f.FuzzMap(target_map) // Fills a map
createdTarBytes, err := f.TarBytes() // Gets bytes of a valid tar archive
err := f.CreateFiles(inThisDir) // Fills inThisDir with files
createdString, err := f.GetStringFrom("anyCharInThisString", ofThisLength) // Gets a string that consists of chars from "anyCharInThisString" and has the exact length "ofThisLength"
```
Most APIs are added as they are needed.
## Projects that use go-fuzz-headers
- [runC](https://github.com/opencontainers/runc)
- [Istio](https://github.com/istio/istio)
- [Vitess](https://github.com/vitessio/vitess)
- [Containerd](https://github.com/containerd/containerd)
Feel free to add your own project to the list, if you use go-fuzz-headers to fuzz it.
## Status
The project is under development and will be updated regularly.
## References
go-fuzz-headers' approach to fuzzing structs is strongly inspired by [gofuzz](https://github.com/google/gofuzz).

914
vendor/github.com/AdaLogics/go-fuzz-headers/consumer.go generated vendored Normal file
View File

@ -0,0 +1,914 @@
// Copyright 2023 The go-fuzz-headers Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gofuzzheaders
import (
"archive/tar"
"bytes"
"encoding/binary"
"errors"
"fmt"
"io"
"math"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"time"
"unsafe"
)
var (
MaxTotalLen uint32 = 2000000
maxDepth = 100
)
func SetMaxTotalLen(newLen uint32) {
MaxTotalLen = newLen
}
type ConsumeFuzzer struct {
data []byte
dataTotal uint32
CommandPart []byte
RestOfArray []byte
NumberOfCalls int
position uint32
fuzzUnexportedFields bool
curDepth int
Funcs map[reflect.Type]reflect.Value
}
func IsDivisibleBy(n int, divisibleby int) bool {
return (n % divisibleby) == 0
}
func NewConsumer(fuzzData []byte) *ConsumeFuzzer {
return &ConsumeFuzzer{
data: fuzzData,
dataTotal: uint32(len(fuzzData)),
Funcs: make(map[reflect.Type]reflect.Value),
curDepth: 0,
}
}
func (f *ConsumeFuzzer) Split(minCalls, maxCalls int) error {
if f.dataTotal == 0 {
return errors.New("could not split")
}
numberOfCalls := int(f.data[0])
if numberOfCalls < minCalls || numberOfCalls > maxCalls {
return errors.New("bad number of calls")
}
if int(f.dataTotal) < numberOfCalls+numberOfCalls+1 {
return errors.New("length of data does not match required parameters")
}
// Define part 2 and 3 of the data array
commandPart := f.data[1 : numberOfCalls+1]
restOfArray := f.data[numberOfCalls+1:]
// Just a small check. It is necessary
if len(commandPart) != numberOfCalls {
return errors.New("length of commandPart does not match number of calls")
}
// Check if restOfArray is divisible by numberOfCalls
if !IsDivisibleBy(len(restOfArray), numberOfCalls) {
return errors.New("length of commandPart does not match number of calls")
}
f.CommandPart = commandPart
f.RestOfArray = restOfArray
f.NumberOfCalls = numberOfCalls
return nil
}
func (f *ConsumeFuzzer) AllowUnexportedFields() {
f.fuzzUnexportedFields = true
}
func (f *ConsumeFuzzer) DisallowUnexportedFields() {
f.fuzzUnexportedFields = false
}
func (f *ConsumeFuzzer) GenerateStruct(targetStruct interface{}) error {
e := reflect.ValueOf(targetStruct).Elem()
return f.fuzzStruct(e, false)
}
func (f *ConsumeFuzzer) setCustom(v reflect.Value) error {
// First: see if we have a fuzz function for it.
doCustom, ok := f.Funcs[v.Type()]
if !ok {
return fmt.Errorf("could not find a custom function")
}
switch v.Kind() {
case reflect.Ptr:
if v.IsNil() {
if !v.CanSet() {
return fmt.Errorf("could not use a custom function")
}
v.Set(reflect.New(v.Type().Elem()))
}
case reflect.Map:
if v.IsNil() {
if !v.CanSet() {
return fmt.Errorf("could not use a custom function")
}
v.Set(reflect.MakeMap(v.Type()))
}
default:
return fmt.Errorf("could not use a custom function")
}
verr := doCustom.Call([]reflect.Value{v, reflect.ValueOf(Continue{
F: f,
})})
// check if we return an error
if verr[0].IsNil() {
return nil
}
return fmt.Errorf("could not use a custom function")
}
func (f *ConsumeFuzzer) fuzzStruct(e reflect.Value, customFunctions bool) error {
if f.curDepth >= maxDepth {
// return err or nil here?
return nil
}
f.curDepth++
defer func() { f.curDepth-- }()
// We check if we should check for custom functions
if customFunctions && e.IsValid() && e.CanAddr() {
err := f.setCustom(e.Addr())
if err != nil {
return err
}
}
switch e.Kind() {
case reflect.Struct:
for i := 0; i < e.NumField(); i++ {
var v reflect.Value
if !e.Field(i).CanSet() {
if f.fuzzUnexportedFields {
v = reflect.NewAt(e.Field(i).Type(), unsafe.Pointer(e.Field(i).UnsafeAddr())).Elem()
}
if err := f.fuzzStruct(v, customFunctions); err != nil {
return err
}
} else {
v = e.Field(i)
if err := f.fuzzStruct(v, customFunctions); err != nil {
return err
}
}
}
case reflect.String:
str, err := f.GetString()
if err != nil {
return err
}
if e.CanSet() {
e.SetString(str)
}
case reflect.Slice:
var maxElements uint32
// Byte slices should not be restricted
if e.Type().String() == "[]uint8" {
maxElements = 10000000
} else {
maxElements = 50
}
randQty, err := f.GetUint32()
if err != nil {
return err
}
numOfElements := randQty % maxElements
if (f.dataTotal - f.position) < numOfElements {
numOfElements = f.dataTotal - f.position
}
uu := reflect.MakeSlice(e.Type(), int(numOfElements), int(numOfElements))
for i := 0; i < int(numOfElements); i++ {
// If we have more than 10, then we can proceed with that.
if err := f.fuzzStruct(uu.Index(i), customFunctions); err != nil {
if i >= 10 {
if e.CanSet() {
e.Set(uu)
}
return nil
} else {
return err
}
}
}
if e.CanSet() {
e.Set(uu)
}
case reflect.Uint16:
newInt, err := f.GetUint16()
if err != nil {
return err
}
if e.CanSet() {
e.SetUint(uint64(newInt))
}
case reflect.Uint32:
newInt, err := f.GetUint32()
if err != nil {
return err
}
if e.CanSet() {
e.SetUint(uint64(newInt))
}
case reflect.Uint64:
newInt, err := f.GetInt()
if err != nil {
return err
}
if e.CanSet() {
e.SetUint(uint64(newInt))
}
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
newInt, err := f.GetInt()
if err != nil {
return err
}
if e.CanSet() {
e.SetInt(int64(newInt))
}
case reflect.Float32:
newFloat, err := f.GetFloat32()
if err != nil {
return err
}
if e.CanSet() {
e.SetFloat(float64(newFloat))
}
case reflect.Float64:
newFloat, err := f.GetFloat64()
if err != nil {
return err
}
if e.CanSet() {
e.SetFloat(float64(newFloat))
}
case reflect.Map:
if e.CanSet() {
e.Set(reflect.MakeMap(e.Type()))
const maxElements = 50
randQty, err := f.GetInt()
if err != nil {
return err
}
numOfElements := randQty % maxElements
for i := 0; i < numOfElements; i++ {
key := reflect.New(e.Type().Key()).Elem()
if err := f.fuzzStruct(key, customFunctions); err != nil {
return err
}
val := reflect.New(e.Type().Elem()).Elem()
if err = f.fuzzStruct(val, customFunctions); err != nil {
return err
}
e.SetMapIndex(key, val)
}
}
case reflect.Ptr:
if e.CanSet() {
e.Set(reflect.New(e.Type().Elem()))
if err := f.fuzzStruct(e.Elem(), customFunctions); err != nil {
return err
}
return nil
}
case reflect.Uint8:
b, err := f.GetByte()
if err != nil {
return err
}
if e.CanSet() {
e.SetUint(uint64(b))
}
}
return nil
}
func (f *ConsumeFuzzer) GetStringArray() (reflect.Value, error) {
// The max size of the array:
const max uint32 = 20
arraySize := f.position
if arraySize > max {
arraySize = max
}
stringArray := reflect.MakeSlice(reflect.SliceOf(reflect.TypeOf("string")), int(arraySize), int(arraySize))
if f.position+arraySize >= f.dataTotal {
return stringArray, errors.New("could not make string array")
}
for i := 0; i < int(arraySize); i++ {
stringSize := uint32(f.data[f.position])
if f.position+stringSize >= f.dataTotal {
return stringArray, nil
}
stringToAppend := string(f.data[f.position : f.position+stringSize])
strVal := reflect.ValueOf(stringToAppend)
stringArray = reflect.Append(stringArray, strVal)
f.position += stringSize
}
return stringArray, nil
}
func (f *ConsumeFuzzer) GetInt() (int, error) {
if f.position >= f.dataTotal {
return 0, errors.New("not enough bytes to create int")
}
returnInt := int(f.data[f.position])
f.position++
return returnInt, nil
}
func (f *ConsumeFuzzer) GetByte() (byte, error) {
if f.position >= f.dataTotal {
return 0x00, errors.New("not enough bytes to get byte")
}
returnByte := f.data[f.position]
f.position++
return returnByte, nil
}
func (f *ConsumeFuzzer) GetNBytes(numberOfBytes int) ([]byte, error) {
if f.position >= f.dataTotal {
return nil, errors.New("not enough bytes to get byte")
}
returnBytes := make([]byte, 0, numberOfBytes)
for i := 0; i < numberOfBytes; i++ {
newByte, err := f.GetByte()
if err != nil {
return nil, err
}
returnBytes = append(returnBytes, newByte)
}
return returnBytes, nil
}
func (f *ConsumeFuzzer) GetUint16() (uint16, error) {
u16, err := f.GetNBytes(2)
if err != nil {
return 0, err
}
littleEndian, err := f.GetBool()
if err != nil {
return 0, err
}
if littleEndian {
return binary.LittleEndian.Uint16(u16), nil
}
return binary.BigEndian.Uint16(u16), nil
}
func (f *ConsumeFuzzer) GetUint32() (uint32, error) {
u32, err := f.GetNBytes(4)
if err != nil {
return 0, err
}
return binary.BigEndian.Uint32(u32), nil
}
func (f *ConsumeFuzzer) GetUint64() (uint64, error) {
u64, err := f.GetNBytes(8)
if err != nil {
return 0, err
}
littleEndian, err := f.GetBool()
if err != nil {
return 0, err
}
if littleEndian {
return binary.LittleEndian.Uint64(u64), nil
}
return binary.BigEndian.Uint64(u64), nil
}
func (f *ConsumeFuzzer) GetBytes() ([]byte, error) {
var length uint32
var err error
length, err = f.GetUint32()
if err != nil {
return nil, errors.New("not enough bytes to create byte array")
}
if length == 0 {
length = 30
}
bytesLeft := f.dataTotal - f.position
if bytesLeft <= 0 {
return nil, errors.New("not enough bytes to create byte array")
}
// If the length is the same as bytes left, we will not overflow
// the remaining bytes.
if length != bytesLeft {
length = length % bytesLeft
}
byteBegin := f.position
if byteBegin+length < byteBegin {
return nil, errors.New("numbers overflow")
}
f.position = byteBegin + length
return f.data[byteBegin:f.position], nil
}
func (f *ConsumeFuzzer) GetString() (string, error) {
if f.position >= f.dataTotal {
return "nil", errors.New("not enough bytes to create string")
}
length, err := f.GetUint32()
if err != nil {
return "nil", errors.New("not enough bytes to create string")
}
if f.position > MaxTotalLen {
return "nil", errors.New("created too large a string")
}
byteBegin := f.position
if byteBegin >= f.dataTotal {
return "nil", errors.New("not enough bytes to create string")
}
if byteBegin+length > f.dataTotal {
return "nil", errors.New("not enough bytes to create string")
}
if byteBegin > byteBegin+length {
return "nil", errors.New("numbers overflow")
}
f.position = byteBegin + length
return string(f.data[byteBegin:f.position]), nil
}
func (f *ConsumeFuzzer) GetBool() (bool, error) {
if f.position >= f.dataTotal {
return false, errors.New("not enough bytes to create bool")
}
if IsDivisibleBy(int(f.data[f.position]), 2) {
f.position++
return true, nil
} else {
f.position++
return false, nil
}
}
func (f *ConsumeFuzzer) FuzzMap(m interface{}) error {
return f.GenerateStruct(m)
}
func returnTarBytes(buf []byte) ([]byte, error) {
return buf, nil
// Count files
var fileCounter int
tr := tar.NewReader(bytes.NewReader(buf))
for {
_, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
fileCounter++
}
if fileCounter >= 1 {
return buf, nil
}
return nil, fmt.Errorf("not enough files were created\n")
}
func setTarHeaderFormat(hdr *tar.Header, f *ConsumeFuzzer) error {
ind, err := f.GetInt()
if err != nil {
hdr.Format = tar.FormatGNU
//return nil
}
switch ind % 4 {
case 0:
hdr.Format = tar.FormatUnknown
case 1:
hdr.Format = tar.FormatUSTAR
case 2:
hdr.Format = tar.FormatPAX
case 3:
hdr.Format = tar.FormatGNU
}
return nil
}
func setTarHeaderTypeflag(hdr *tar.Header, f *ConsumeFuzzer) error {
ind, err := f.GetInt()
if err != nil {
return err
}
switch ind % 13 {
case 0:
hdr.Typeflag = tar.TypeReg
case 1:
hdr.Typeflag = tar.TypeLink
linkname, err := f.GetString()
if err != nil {
return err
}
hdr.Linkname = linkname
case 2:
hdr.Typeflag = tar.TypeSymlink
linkname, err := f.GetString()
if err != nil {
return err
}
hdr.Linkname = linkname
case 3:
hdr.Typeflag = tar.TypeChar
case 4:
hdr.Typeflag = tar.TypeBlock
case 5:
hdr.Typeflag = tar.TypeDir
case 6:
hdr.Typeflag = tar.TypeFifo
case 7:
hdr.Typeflag = tar.TypeCont
case 8:
hdr.Typeflag = tar.TypeXHeader
case 9:
hdr.Typeflag = tar.TypeXGlobalHeader
case 10:
hdr.Typeflag = tar.TypeGNUSparse
case 11:
hdr.Typeflag = tar.TypeGNULongName
case 12:
hdr.Typeflag = tar.TypeGNULongLink
}
return nil
}
func (f *ConsumeFuzzer) createTarFileBody() ([]byte, error) {
return f.GetBytes()
/*length, err := f.GetUint32()
if err != nil {
return nil, errors.New("not enough bytes to create byte array")
}
// A bit of optimization to attempt to create a file body
// when we don't have as many bytes left as "length"
remainingBytes := f.dataTotal - f.position
if remainingBytes <= 0 {
return nil, errors.New("created too large a string")
}
if f.position+length > MaxTotalLen {
return nil, errors.New("created too large a string")
}
byteBegin := f.position
if byteBegin >= f.dataTotal {
return nil, errors.New("not enough bytes to create byte array")
}
if length == 0 {
return nil, errors.New("zero-length is not supported")
}
if byteBegin+length >= f.dataTotal {
return nil, errors.New("not enough bytes to create byte array")
}
if byteBegin+length < byteBegin {
return nil, errors.New("numbers overflow")
}
f.position = byteBegin + length
return f.data[byteBegin:f.position], nil*/
}
// getTarFileName is similar to GetString(), but creates string based
// on the length of f.data to reduce the likelihood of overflowing
// f.data.
func (f *ConsumeFuzzer) getTarFilename() (string, error) {
return f.GetString()
/*length, err := f.GetUint32()
if err != nil {
return "nil", errors.New("not enough bytes to create string")
}
// A bit of optimization to attempt to create a file name
// when we don't have as many bytes left as "length"
remainingBytes := f.dataTotal - f.position
if remainingBytes <= 0 {
return "nil", errors.New("created too large a string")
}
if f.position > MaxTotalLen {
return "nil", errors.New("created too large a string")
}
byteBegin := f.position
if byteBegin >= f.dataTotal {
return "nil", errors.New("not enough bytes to create string")
}
if byteBegin+length > f.dataTotal {
return "nil", errors.New("not enough bytes to create string")
}
if byteBegin > byteBegin+length {
return "nil", errors.New("numbers overflow")
}
f.position = byteBegin + length
return string(f.data[byteBegin:f.position]), nil*/
}
type TarFile struct {
Hdr *tar.Header
Body []byte
}
// TarBytes returns valid bytes for a tar archive
func (f *ConsumeFuzzer) TarBytes() ([]byte, error) {
numberOfFiles, err := f.GetInt()
if err != nil {
return nil, err
}
var tarFiles []*TarFile
tarFiles = make([]*TarFile, 0)
const maxNoOfFiles = 100
for i := 0; i < numberOfFiles%maxNoOfFiles; i++ {
var filename string
var filebody []byte
var sec, nsec int
var err error
filename, err = f.getTarFilename()
if err != nil {
var sb strings.Builder
sb.WriteString("file-")
sb.WriteString(strconv.Itoa(i))
filename = sb.String()
}
filebody, err = f.createTarFileBody()
if err != nil {
var sb strings.Builder
sb.WriteString("filebody-")
sb.WriteString(strconv.Itoa(i))
filebody = []byte(sb.String())
}
sec, err = f.GetInt()
if err != nil {
sec = 1672531200 // beginning of 2023
}
nsec, err = f.GetInt()
if err != nil {
nsec = 1703980800 // end of 2023
}
hdr := &tar.Header{
Name: filename,
Size: int64(len(filebody)),
Mode: 0o600,
ModTime: time.Unix(int64(sec), int64(nsec)),
}
if err := setTarHeaderTypeflag(hdr, f); err != nil {
return []byte(""), err
}
if err := setTarHeaderFormat(hdr, f); err != nil {
return []byte(""), err
}
tf := &TarFile{
Hdr: hdr,
Body: filebody,
}
tarFiles = append(tarFiles, tf)
}
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
defer tw.Close()
for _, tf := range tarFiles {
tw.WriteHeader(tf.Hdr)
tw.Write(tf.Body)
}
return buf.Bytes(), nil
}
// This is similar to TarBytes, but it returns a series of
// files instead of raw tar bytes. The advantage of this
// api is that it is cheaper in terms of cpu power to
// modify or check the files in the fuzzer with TarFiles()
// because it avoids creating a tar reader.
func (f *ConsumeFuzzer) TarFiles() ([]*TarFile, error) {
numberOfFiles, err := f.GetInt()
if err != nil {
return nil, err
}
var tarFiles []*TarFile
tarFiles = make([]*TarFile, 0)
const maxNoOfFiles = 100
for i := 0; i < numberOfFiles%maxNoOfFiles; i++ {
filename, err := f.getTarFilename()
if err != nil {
return tarFiles, err
}
filebody, err := f.createTarFileBody()
if err != nil {
return tarFiles, err
}
sec, err := f.GetInt()
if err != nil {
return tarFiles, err
}
nsec, err := f.GetInt()
if err != nil {
return tarFiles, err
}
hdr := &tar.Header{
Name: filename,
Size: int64(len(filebody)),
Mode: 0o600,
ModTime: time.Unix(int64(sec), int64(nsec)),
}
if err := setTarHeaderTypeflag(hdr, f); err != nil {
hdr.Typeflag = tar.TypeReg
}
if err := setTarHeaderFormat(hdr, f); err != nil {
return tarFiles, err // should not happend
}
tf := &TarFile{
Hdr: hdr,
Body: filebody,
}
tarFiles = append(tarFiles, tf)
}
return tarFiles, nil
}
// CreateFiles creates pseudo-random files in rootDir.
// It creates subdirs and places the files there.
// It is the callers responsibility to ensure that
// rootDir exists.
func (f *ConsumeFuzzer) CreateFiles(rootDir string) error {
numberOfFiles, err := f.GetInt()
if err != nil {
return err
}
maxNumberOfFiles := numberOfFiles % 4000 // This is completely arbitrary
if maxNumberOfFiles == 0 {
return errors.New("maxNumberOfFiles is nil")
}
var noOfCreatedFiles int
for i := 0; i < maxNumberOfFiles; i++ {
// The file to create:
fileName, err := f.GetString()
if err != nil {
if noOfCreatedFiles > 0 {
// If files have been created, we don't return an error.
break
} else {
return errors.New("could not get fileName")
}
}
if strings.Contains(fileName, "..") || (len(fileName) > 0 && fileName[0] == 47) || strings.Contains(fileName, "\\") {
continue
}
fullFilePath := filepath.Join(rootDir, fileName)
// Find the subdirectory of the file
if subDir := filepath.Dir(fileName); subDir != "" && subDir != "." {
// create the dir first; avoid going outside the root dir
if strings.Contains(subDir, "../") || (len(subDir) > 0 && subDir[0] == 47) || strings.Contains(subDir, "\\") {
continue
}
dirPath := filepath.Join(rootDir, subDir)
if _, err := os.Stat(dirPath); os.IsNotExist(err) {
err2 := os.MkdirAll(dirPath, 0o777)
if err2 != nil {
continue
}
}
fullFilePath = filepath.Join(dirPath, fileName)
} else {
// Create symlink
createSymlink, err := f.GetBool()
if err != nil {
if noOfCreatedFiles > 0 {
break
} else {
return errors.New("could not create the symlink")
}
}
if createSymlink {
symlinkTarget, err := f.GetString()
if err != nil {
return err
}
err = os.Symlink(symlinkTarget, fullFilePath)
if err != nil {
return err
}
// stop loop here, since a symlink needs no further action
noOfCreatedFiles++
continue
}
// We create a normal file
fileContents, err := f.GetBytes()
if err != nil {
if noOfCreatedFiles > 0 {
break
} else {
return errors.New("could not create the file")
}
}
err = os.WriteFile(fullFilePath, fileContents, 0o666)
if err != nil {
continue
}
noOfCreatedFiles++
}
}
return nil
}
// GetStringFrom returns a string that can only consist of characters
// included in possibleChars. It returns an error if the created string
// does not have the specified length.
func (f *ConsumeFuzzer) GetStringFrom(possibleChars string, length int) (string, error) {
if (f.dataTotal - f.position) < uint32(length) {
return "", errors.New("not enough bytes to create a string")
}
output := make([]byte, 0, length)
for i := 0; i < length; i++ {
charIndex, err := f.GetInt()
if err != nil {
return string(output), err
}
output = append(output, possibleChars[charIndex%len(possibleChars)])
}
return string(output), nil
}
func (f *ConsumeFuzzer) GetRune() ([]rune, error) {
stringToConvert, err := f.GetString()
if err != nil {
return []rune("nil"), err
}
return []rune(stringToConvert), nil
}
func (f *ConsumeFuzzer) GetFloat32() (float32, error) {
u32, err := f.GetNBytes(4)
if err != nil {
return 0, err
}
littleEndian, err := f.GetBool()
if err != nil {
return 0, err
}
if littleEndian {
u32LE := binary.LittleEndian.Uint32(u32)
return math.Float32frombits(u32LE), nil
}
u32BE := binary.BigEndian.Uint32(u32)
return math.Float32frombits(u32BE), nil
}
func (f *ConsumeFuzzer) GetFloat64() (float64, error) {
u64, err := f.GetNBytes(8)
if err != nil {
return 0, err
}
littleEndian, err := f.GetBool()
if err != nil {
return 0, err
}
if littleEndian {
u64LE := binary.LittleEndian.Uint64(u64)
return math.Float64frombits(u64LE), nil
}
u64BE := binary.BigEndian.Uint64(u64)
return math.Float64frombits(u64BE), nil
}
func (f *ConsumeFuzzer) CreateSlice(targetSlice interface{}) error {
return f.GenerateStruct(targetSlice)
}

62
vendor/github.com/AdaLogics/go-fuzz-headers/funcs.go generated vendored Normal file
View File

@ -0,0 +1,62 @@
// Copyright 2023 The go-fuzz-headers Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gofuzzheaders
import (
"fmt"
"reflect"
)
type Continue struct {
F *ConsumeFuzzer
}
func (f *ConsumeFuzzer) AddFuncs(fuzzFuncs []interface{}) {
for i := range fuzzFuncs {
v := reflect.ValueOf(fuzzFuncs[i])
if v.Kind() != reflect.Func {
panic("Need only funcs!")
}
t := v.Type()
if t.NumIn() != 2 || t.NumOut() != 1 {
fmt.Println(t.NumIn(), t.NumOut())
panic("Need 2 in and 1 out params. In must be the type. Out must be an error")
}
argT := t.In(0)
switch argT.Kind() {
case reflect.Ptr, reflect.Map:
default:
panic("fuzzFunc must take pointer or map type")
}
if t.In(1) != reflect.TypeOf(Continue{}) {
panic("fuzzFunc's second parameter must be type Continue")
}
f.Funcs[argT] = v
}
}
func (f *ConsumeFuzzer) GenerateWithCustom(targetStruct interface{}) error {
e := reflect.ValueOf(targetStruct).Elem()
return f.fuzzStruct(e, true)
}
func (c Continue) GenerateStruct(targetStruct interface{}) error {
return c.F.GenerateStruct(targetStruct)
}
func (c Continue) GenerateStructWithCustom(targetStruct interface{}) error {
return c.F.GenerateWithCustom(targetStruct)
}

556
vendor/github.com/AdaLogics/go-fuzz-headers/sql.go generated vendored Normal file
View File

@ -0,0 +1,556 @@
// Copyright 2023 The go-fuzz-headers Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package gofuzzheaders
import (
"fmt"
"strings"
)
// returns a keyword by index
func getKeyword(f *ConsumeFuzzer) (string, error) {
index, err := f.GetInt()
if err != nil {
return keywords[0], err
}
for i, k := range keywords {
if i == index {
return k, nil
}
}
return keywords[0], fmt.Errorf("could not get a kw")
}
// Simple utility function to check if a string
// slice contains a string.
func containsString(s []string, e string) bool {
for _, a := range s {
if a == e {
return true
}
}
return false
}
// These keywords are used specifically for fuzzing Vitess
var keywords = []string{
"accessible", "action", "add", "after", "against", "algorithm",
"all", "alter", "always", "analyze", "and", "as", "asc", "asensitive",
"auto_increment", "avg_row_length", "before", "begin", "between",
"bigint", "binary", "_binary", "_utf8mb4", "_utf8", "_latin1", "bit",
"blob", "bool", "boolean", "both", "by", "call", "cancel", "cascade",
"cascaded", "case", "cast", "channel", "change", "char", "character",
"charset", "check", "checksum", "coalesce", "code", "collate", "collation",
"column", "columns", "comment", "committed", "commit", "compact", "complete",
"compressed", "compression", "condition", "connection", "constraint", "continue",
"convert", "copy", "cume_dist", "substr", "substring", "create", "cross",
"csv", "current_date", "current_time", "current_timestamp", "current_user",
"cursor", "data", "database", "databases", "day", "day_hour", "day_microsecond",
"day_minute", "day_second", "date", "datetime", "dec", "decimal", "declare",
"default", "definer", "delay_key_write", "delayed", "delete", "dense_rank",
"desc", "describe", "deterministic", "directory", "disable", "discard",
"disk", "distinct", "distinctrow", "div", "double", "do", "drop", "dumpfile",
"duplicate", "dynamic", "each", "else", "elseif", "empty", "enable",
"enclosed", "encryption", "end", "enforced", "engine", "engines", "enum",
"error", "escape", "escaped", "event", "exchange", "exclusive", "exists",
"exit", "explain", "expansion", "export", "extended", "extract", "false",
"fetch", "fields", "first", "first_value", "fixed", "float", "float4",
"float8", "flush", "for", "force", "foreign", "format", "from", "full",
"fulltext", "function", "general", "generated", "geometry", "geometrycollection",
"get", "global", "gtid_executed", "grant", "group", "grouping", "groups",
"group_concat", "having", "header", "high_priority", "hosts", "hour", "hour_microsecond",
"hour_minute", "hour_second", "if", "ignore", "import", "in", "index", "indexes",
"infile", "inout", "inner", "inplace", "insensitive", "insert", "insert_method",
"int", "int1", "int2", "int3", "int4", "int8", "integer", "interval",
"into", "io_after_gtids", "is", "isolation", "iterate", "invoker", "join",
"json", "json_table", "key", "keys", "keyspaces", "key_block_size", "kill", "lag",
"language", "last", "last_value", "last_insert_id", "lateral", "lead", "leading",
"leave", "left", "less", "level", "like", "limit", "linear", "lines",
"linestring", "load", "local", "localtime", "localtimestamp", "lock", "logs",
"long", "longblob", "longtext", "loop", "low_priority", "manifest",
"master_bind", "match", "max_rows", "maxvalue", "mediumblob", "mediumint",
"mediumtext", "memory", "merge", "microsecond", "middleint", "min_rows", "minute",
"minute_microsecond", "minute_second", "mod", "mode", "modify", "modifies",
"multilinestring", "multipoint", "multipolygon", "month", "name",
"names", "natural", "nchar", "next", "no", "none", "not", "no_write_to_binlog",
"nth_value", "ntile", "null", "numeric", "of", "off", "offset", "on",
"only", "open", "optimize", "optimizer_costs", "option", "optionally",
"or", "order", "out", "outer", "outfile", "over", "overwrite", "pack_keys",
"parser", "partition", "partitioning", "password", "percent_rank", "plugins",
"point", "polygon", "precision", "primary", "privileges", "processlist",
"procedure", "query", "quarter", "range", "rank", "read", "reads", "read_write",
"real", "rebuild", "recursive", "redundant", "references", "regexp", "relay",
"release", "remove", "rename", "reorganize", "repair", "repeat", "repeatable",
"replace", "require", "resignal", "restrict", "return", "retry", "revert",
"revoke", "right", "rlike", "rollback", "row", "row_format", "row_number",
"rows", "s3", "savepoint", "schema", "schemas", "second", "second_microsecond",
"security", "select", "sensitive", "separator", "sequence", "serializable",
"session", "set", "share", "shared", "show", "signal", "signed", "slow",
"smallint", "spatial", "specific", "sql", "sqlexception", "sqlstate",
"sqlwarning", "sql_big_result", "sql_cache", "sql_calc_found_rows",
"sql_no_cache", "sql_small_result", "ssl", "start", "starting",
"stats_auto_recalc", "stats_persistent", "stats_sample_pages", "status",
"storage", "stored", "straight_join", "stream", "system", "vstream",
"table", "tables", "tablespace", "temporary", "temptable", "terminated",
"text", "than", "then", "time", "timestamp", "timestampadd", "timestampdiff",
"tinyblob", "tinyint", "tinytext", "to", "trailing", "transaction", "tree",
"traditional", "trigger", "triggers", "true", "truncate", "uncommitted",
"undefined", "undo", "union", "unique", "unlock", "unsigned", "update",
"upgrade", "usage", "use", "user", "user_resources", "using", "utc_date",
"utc_time", "utc_timestamp", "validation", "values", "variables", "varbinary",
"varchar", "varcharacter", "varying", "vgtid_executed", "virtual", "vindex",
"vindexes", "view", "vitess", "vitess_keyspaces", "vitess_metadata",
"vitess_migration", "vitess_migrations", "vitess_replication_status",
"vitess_shards", "vitess_tablets", "vschema", "warnings", "when",
"where", "while", "window", "with", "without", "work", "write", "xor",
"year", "year_month", "zerofill",
}
// Keywords that could get an additional keyword
var needCustomString = []string{
"DISTINCTROW", "FROM", // Select keywords:
"GROUP BY", "HAVING", "WINDOW",
"FOR",
"ORDER BY", "LIMIT",
"INTO", "PARTITION", "AS", // Insert Keywords:
"ON DUPLICATE KEY UPDATE",
"WHERE", "LIMIT", // Delete keywords
"INFILE", "INTO TABLE", "CHARACTER SET", // Load keywords
"TERMINATED BY", "ENCLOSED BY",
"ESCAPED BY", "STARTING BY",
"TERMINATED BY", "STARTING BY",
"IGNORE",
"VALUE", "VALUES", // Replace tokens
"SET", // Update tokens
"ENGINE =", // Drop tokens
"DEFINER =", "ON SCHEDULE", "RENAME TO", // Alter tokens
"COMMENT", "DO", "INITIAL_SIZE = ", "OPTIONS",
}
var alterTableTokens = [][]string{
{"CUSTOM_FUZZ_STRING"},
{"CUSTOM_ALTTER_TABLE_OPTIONS"},
{"PARTITION_OPTIONS_FOR_ALTER_TABLE"},
}
var alterTokens = [][]string{
{
"DATABASE", "SCHEMA", "DEFINER = ", "EVENT", "FUNCTION", "INSTANCE",
"LOGFILE GROUP", "PROCEDURE", "SERVER",
},
{"CUSTOM_FUZZ_STRING"},
{
"ON SCHEDULE", "ON COMPLETION PRESERVE", "ON COMPLETION NOT PRESERVE",
"ADD UNDOFILE", "OPTIONS",
},
{"RENAME TO", "INITIAL_SIZE = "},
{"ENABLE", "DISABLE", "DISABLE ON SLAVE", "ENGINE"},
{"COMMENT"},
{"DO"},
}
var setTokens = [][]string{
{"CHARACTER SET", "CHARSET", "CUSTOM_FUZZ_STRING", "NAMES"},
{"CUSTOM_FUZZ_STRING", "DEFAULT", "="},
{"CUSTOM_FUZZ_STRING"},
}
var dropTokens = [][]string{
{"TEMPORARY", "UNDO"},
{
"DATABASE", "SCHEMA", "EVENT", "INDEX", "LOGFILE GROUP",
"PROCEDURE", "FUNCTION", "SERVER", "SPATIAL REFERENCE SYSTEM",
"TABLE", "TABLESPACE", "TRIGGER", "VIEW",
},
{"IF EXISTS"},
{"CUSTOM_FUZZ_STRING"},
{"ON", "ENGINE = ", "RESTRICT", "CASCADE"},
}
var renameTokens = [][]string{
{"TABLE"},
{"CUSTOM_FUZZ_STRING"},
{"TO"},
{"CUSTOM_FUZZ_STRING"},
}
var truncateTokens = [][]string{
{"TABLE"},
{"CUSTOM_FUZZ_STRING"},
}
var createTokens = [][]string{
{"OR REPLACE", "TEMPORARY", "UNDO"}, // For create spatial reference system
{
"UNIQUE", "FULLTEXT", "SPATIAL", "ALGORITHM = UNDEFINED", "ALGORITHM = MERGE",
"ALGORITHM = TEMPTABLE",
},
{
"DATABASE", "SCHEMA", "EVENT", "FUNCTION", "INDEX", "LOGFILE GROUP",
"PROCEDURE", "SERVER", "SPATIAL REFERENCE SYSTEM", "TABLE", "TABLESPACE",
"TRIGGER", "VIEW",
},
{"IF NOT EXISTS"},
{"CUSTOM_FUZZ_STRING"},
}
/*
// For future use.
var updateTokens = [][]string{
{"LOW_PRIORITY"},
{"IGNORE"},
{"SET"},
{"WHERE"},
{"ORDER BY"},
{"LIMIT"},
}
*/
var replaceTokens = [][]string{
{"LOW_PRIORITY", "DELAYED"},
{"INTO"},
{"PARTITION"},
{"CUSTOM_FUZZ_STRING"},
{"VALUES", "VALUE"},
}
var loadTokens = [][]string{
{"DATA"},
{"LOW_PRIORITY", "CONCURRENT", "LOCAL"},
{"INFILE"},
{"REPLACE", "IGNORE"},
{"INTO TABLE"},
{"PARTITION"},
{"CHARACTER SET"},
{"FIELDS", "COLUMNS"},
{"TERMINATED BY"},
{"OPTIONALLY"},
{"ENCLOSED BY"},
{"ESCAPED BY"},
{"LINES"},
{"STARTING BY"},
{"TERMINATED BY"},
{"IGNORE"},
{"LINES", "ROWS"},
{"CUSTOM_FUZZ_STRING"},
}
// These Are everything that comes after "INSERT"
var insertTokens = [][]string{
{"LOW_PRIORITY", "DELAYED", "HIGH_PRIORITY", "IGNORE"},
{"INTO"},
{"PARTITION"},
{"CUSTOM_FUZZ_STRING"},
{"AS"},
{"ON DUPLICATE KEY UPDATE"},
}
// These are everything that comes after "SELECT"
var selectTokens = [][]string{
{"*", "CUSTOM_FUZZ_STRING", "DISTINCTROW"},
{"HIGH_PRIORITY"},
{"STRAIGHT_JOIN"},
{"SQL_SMALL_RESULT", "SQL_BIG_RESULT", "SQL_BUFFER_RESULT"},
{"SQL_NO_CACHE", "SQL_CALC_FOUND_ROWS"},
{"CUSTOM_FUZZ_STRING"},
{"FROM"},
{"WHERE"},
{"GROUP BY"},
{"HAVING"},
{"WINDOW"},
{"ORDER BY"},
{"LIMIT"},
{"CUSTOM_FUZZ_STRING"},
{"FOR"},
}
// These are everything that comes after "DELETE"
var deleteTokens = [][]string{
{"LOW_PRIORITY", "QUICK", "IGNORE", "FROM", "AS"},
{"PARTITION"},
{"WHERE"},
{"ORDER BY"},
{"LIMIT"},
}
var alter_table_options = []string{
"ADD", "COLUMN", "FIRST", "AFTER", "INDEX", "KEY", "FULLTEXT", "SPATIAL",
"CONSTRAINT", "UNIQUE", "FOREIGN KEY", "CHECK", "ENFORCED", "DROP", "ALTER",
"NOT", "INPLACE", "COPY", "SET", "VISIBLE", "INVISIBLE", "DEFAULT", "CHANGE",
"CHARACTER SET", "COLLATE", "DISABLE", "ENABLE", "KEYS", "TABLESPACE", "LOCK",
"FORCE", "MODIFY", "SHARED", "EXCLUSIVE", "NONE", "ORDER BY", "RENAME COLUMN",
"AS", "=", "ASC", "DESC", "WITH", "WITHOUT", "VALIDATION", "ADD PARTITION",
"DROP PARTITION", "DISCARD PARTITION", "IMPORT PARTITION", "TRUNCATE PARTITION",
"COALESCE PARTITION", "REORGANIZE PARTITION", "EXCHANGE PARTITION",
"ANALYZE PARTITION", "CHECK PARTITION", "OPTIMIZE PARTITION", "REBUILD PARTITION",
"REPAIR PARTITION", "REMOVE PARTITIONING", "USING", "BTREE", "HASH", "COMMENT",
"KEY_BLOCK_SIZE", "WITH PARSER", "AUTOEXTEND_SIZE", "AUTO_INCREMENT", "AVG_ROW_LENGTH",
"CHECKSUM", "INSERT_METHOD", "ROW_FORMAT", "DYNAMIC", "FIXED", "COMPRESSED", "REDUNDANT",
"COMPACT", "SECONDARY_ENGINE_ATTRIBUTE", "STATS_AUTO_RECALC", "STATS_PERSISTENT",
"STATS_SAMPLE_PAGES", "ZLIB", "LZ4", "ENGINE_ATTRIBUTE", "KEY_BLOCK_SIZE", "MAX_ROWS",
"MIN_ROWS", "PACK_KEYS", "PASSWORD", "COMPRESSION", "CONNECTION", "DIRECTORY",
"DELAY_KEY_WRITE", "ENCRYPTION", "STORAGE", "DISK", "MEMORY", "UNION",
}
// Creates an 'alter table' statement. 'alter table' is an exception
// in that it has its own function. The majority of statements
// are created by 'createStmt()'.
func createAlterTableStmt(f *ConsumeFuzzer) (string, error) {
maxArgs, err := f.GetInt()
if err != nil {
return "", err
}
maxArgs = maxArgs % 30
if maxArgs == 0 {
return "", fmt.Errorf("could not create alter table stmt")
}
var stmt strings.Builder
stmt.WriteString("ALTER TABLE ")
for i := 0; i < maxArgs; i++ {
// Calculate if we get existing token or custom string
tokenType, err := f.GetInt()
if err != nil {
return "", err
}
if tokenType%4 == 1 {
customString, err := f.GetString()
if err != nil {
return "", err
}
stmt.WriteString(" " + customString)
} else {
tokenIndex, err := f.GetInt()
if err != nil {
return "", err
}
stmt.WriteString(" " + alter_table_options[tokenIndex%len(alter_table_options)])
}
}
return stmt.String(), nil
}
func chooseToken(tokens []string, f *ConsumeFuzzer) (string, error) {
index, err := f.GetInt()
if err != nil {
return "", err
}
var token strings.Builder
token.WriteString(tokens[index%len(tokens)])
if token.String() == "CUSTOM_FUZZ_STRING" {
customFuzzString, err := f.GetString()
if err != nil {
return "", err
}
return customFuzzString, nil
}
// Check if token requires an argument
if containsString(needCustomString, token.String()) {
customFuzzString, err := f.GetString()
if err != nil {
return "", err
}
token.WriteString(" " + customFuzzString)
}
return token.String(), nil
}
var stmtTypes = map[string][][]string{
"DELETE": deleteTokens,
"INSERT": insertTokens,
"SELECT": selectTokens,
"LOAD": loadTokens,
"REPLACE": replaceTokens,
"CREATE": createTokens,
"DROP": dropTokens,
"RENAME": renameTokens,
"TRUNCATE": truncateTokens,
"SET": setTokens,
"ALTER": alterTokens,
"ALTER TABLE": alterTableTokens, // ALTER TABLE has its own set of tokens
}
var stmtTypeEnum = map[int]string{
0: "DELETE",
1: "INSERT",
2: "SELECT",
3: "LOAD",
4: "REPLACE",
5: "CREATE",
6: "DROP",
7: "RENAME",
8: "TRUNCATE",
9: "SET",
10: "ALTER",
11: "ALTER TABLE",
}
func createStmt(f *ConsumeFuzzer) (string, error) {
stmtIndex, err := f.GetInt()
if err != nil {
return "", err
}
stmtIndex = stmtIndex % len(stmtTypes)
queryType := stmtTypeEnum[stmtIndex]
tokens := stmtTypes[queryType]
// We have custom creator for ALTER TABLE
if queryType == "ALTER TABLE" {
query, err := createAlterTableStmt(f)
if err != nil {
return "", err
}
return query, nil
}
// Here we are creating a query that is not
// an 'alter table' query. For available
// queries, see "stmtTypes"
// First specify the first query keyword:
var query strings.Builder
query.WriteString(queryType)
// Next create the args for the
queryArgs, err := createStmtArgs(tokens, f)
if err != nil {
return "", err
}
query.WriteString(" " + queryArgs)
return query.String(), nil
}
// Creates the arguments of a statements. In a select statement
// that would be everything after "select".
func createStmtArgs(tokenslice [][]string, f *ConsumeFuzzer) (string, error) {
var query, token strings.Builder
// We go through the tokens in the tokenslice,
// create the respective token and add it to
// "query"
for _, tokens := range tokenslice {
// For extra randomization, the fuzzer can
// choose to not include this token.
includeThisToken, err := f.GetBool()
if err != nil {
return "", err
}
if !includeThisToken {
continue
}
// There may be several tokens to choose from:
if len(tokens) > 1 {
chosenToken, err := chooseToken(tokens, f)
if err != nil {
return "", err
}
query.WriteString(" " + chosenToken)
} else {
token.WriteString(tokens[0])
// In case the token is "CUSTOM_FUZZ_STRING"
// we will then create a non-structured string
if token.String() == "CUSTOM_FUZZ_STRING" {
customFuzzString, err := f.GetString()
if err != nil {
return "", err
}
query.WriteString(" " + customFuzzString)
continue
}
// Check if token requires an argument.
// Tokens that take an argument can be found
// in 'needCustomString'. If so, we add a
// non-structured string to the token.
if containsString(needCustomString, token.String()) {
customFuzzString, err := f.GetString()
if err != nil {
return "", err
}
token.WriteString(fmt.Sprintf(" %s", customFuzzString))
}
query.WriteString(fmt.Sprintf(" %s", token.String()))
}
}
return query.String(), nil
}
// Creates a semi-structured query. It creates a string
// that is a combination of the keywords and random strings.
func createQuery(f *ConsumeFuzzer) (string, error) {
queryLen, err := f.GetInt()
if err != nil {
return "", err
}
maxLen := queryLen % 60
if maxLen == 0 {
return "", fmt.Errorf("could not create a query")
}
var query strings.Builder
for i := 0; i < maxLen; i++ {
// Get a new token:
useKeyword, err := f.GetBool()
if err != nil {
return "", err
}
if useKeyword {
keyword, err := getKeyword(f)
if err != nil {
return "", err
}
query.WriteString(" " + keyword)
} else {
customString, err := f.GetString()
if err != nil {
return "", err
}
query.WriteString(" " + customString)
}
}
if query.String() == "" {
return "", fmt.Errorf("could not create a query")
}
return query.String(), nil
}
// GetSQLString is the API that users interact with.
//
// Usage:
//
// f := NewConsumer(data)
// sqlString, err := f.GetSQLString()
func (f *ConsumeFuzzer) GetSQLString() (string, error) {
var query string
veryStructured, err := f.GetBool()
if err != nil {
return "", err
}
if veryStructured {
query, err = createStmt(f)
if err != nil {
return "", err
}
} else {
query, err = createQuery(f)
if err != nil {
return "", err
}
}
return query, nil
}

201
vendor/github.com/AdamKorcz/go-118-fuzz-build/LICENSE generated vendored Normal file
View File

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@ -0,0 +1,207 @@
package testing
import (
"fmt"
fuzz "github.com/AdaLogics/go-fuzz-headers"
"os"
"reflect"
)
type F struct {
Data []byte
T *T
FuzzFunc func(*T, any)
}
func (f *F) CleanupTempDirs() {
f.T.CleanupTempDirs()
}
func (f *F) Add(args ...any) {}
func (c *F) Cleanup(f func()) {}
func (c *F) Error(args ...any) {}
func (c *F) Errorf(format string, args ...any) {}
func (f *F) Fail() {}
func (c *F) FailNow() {}
func (c *F) Failed() bool { return false }
func (c *F) Fatal(args ...any) {}
func (c *F) Fatalf(format string, args ...any) {}
func (f *F) Fuzz(ff any) {
// we are assuming that ff is a func.
// TODO: Add a check for UX purposes
fn := reflect.ValueOf(ff)
fnType := fn.Type()
var types []reflect.Type
for i := 1; i < fnType.NumIn(); i++ {
t := fnType.In(i)
types = append(types, t)
}
args := []reflect.Value{reflect.ValueOf(f.T)}
fuzzConsumer := fuzz.NewConsumer(f.Data)
for _, v := range types {
switch v.String() {
case "[]uint8":
b, err := fuzzConsumer.GetBytes()
if err != nil {
return
}
newBytes := reflect.New(v)
newBytes.Elem().SetBytes(b)
args = append(args, newBytes.Elem())
case "string":
s, err := fuzzConsumer.GetString()
if err != nil {
return
}
newString := reflect.New(v)
newString.Elem().SetString(s)
args = append(args, newString.Elem())
case "int":
randInt, err := fuzzConsumer.GetInt()
if err != nil {
return
}
newInt := reflect.New(v)
newInt.Elem().SetInt(int64(randInt))
args = append(args, newInt.Elem())
case "int8":
randInt, err := fuzzConsumer.GetInt()
if err != nil {
return
}
newInt := reflect.New(v)
newInt.Elem().SetInt(int64(randInt))
args = append(args, newInt.Elem())
case "int16":
randInt, err := fuzzConsumer.GetInt()
if err != nil {
return
}
newInt := reflect.New(v)
newInt.Elem().SetInt(int64(randInt))
args = append(args, newInt.Elem())
case "int32":
randInt, err := fuzzConsumer.GetInt()
if err != nil {
return
}
newInt := reflect.New(v)
newInt.Elem().SetInt(int64(randInt))
args = append(args, newInt.Elem())
case "int64":
randInt, err := fuzzConsumer.GetInt()
if err != nil {
return
}
newInt := reflect.New(v)
newInt.Elem().SetInt(int64(randInt))
args = append(args, newInt.Elem())
case "uint":
randInt, err := fuzzConsumer.GetInt()
if err != nil {
return
}
newUint := reflect.New(v)
newUint.Elem().SetUint(uint64(randInt))
args = append(args, newUint.Elem())
case "uint8":
randInt, err := fuzzConsumer.GetInt()
if err != nil {
return
}
newUint := reflect.New(v)
newUint.Elem().SetUint(uint64(randInt))
args = append(args, newUint.Elem())
case "uint16":
randInt, err := fuzzConsumer.GetUint16()
if err != nil {
return
}
newUint16 := reflect.New(v)
newUint16.Elem().SetUint(uint64(randInt))
args = append(args, newUint16.Elem())
case "uint32":
randInt, err := fuzzConsumer.GetUint32()
if err != nil {
return
}
newUint32 := reflect.New(v)
newUint32.Elem().SetUint(uint64(randInt))
args = append(args, newUint32.Elem())
case "uint64":
randInt, err := fuzzConsumer.GetUint64()
if err != nil {
return
}
newUint64 := reflect.New(v)
newUint64.Elem().SetUint(uint64(randInt))
args = append(args, newUint64.Elem())
case "rune":
randRune, err := fuzzConsumer.GetRune()
if err != nil {
return
}
newRune := reflect.New(v)
newRune.Elem().Set(reflect.ValueOf(randRune))
args = append(args, newRune.Elem())
case "float32":
randFloat, err := fuzzConsumer.GetFloat32()
if err != nil {
return
}
newFloat := reflect.New(v)
newFloat.Elem().Set(reflect.ValueOf(randFloat))
args = append(args, newFloat.Elem())
case "float64":
randFloat, err := fuzzConsumer.GetFloat64()
if err != nil {
return
}
newFloat := reflect.New(v)
newFloat.Elem().Set(reflect.ValueOf(randFloat))
args = append(args, newFloat.Elem())
case "bool":
randBool, err := fuzzConsumer.GetBool()
if err != nil {
return
}
newBool := reflect.New(v)
newBool.Elem().Set(reflect.ValueOf(randBool))
args = append(args, newBool.Elem())
default:
fmt.Println(v.String())
}
}
fn.Call(args)
}
func (f *F) Helper() {}
func (c *F) Log(args ...any) {
fmt.Println(args...)
}
func (c *F) Logf(format string, args ...any) {
fmt.Println(format, args)
}
func (c *F) Name() string { return "libFuzzer" }
func (c *F) Setenv(key, value string) {}
func (c *F) Skip(args ...any) {
panic("GO-FUZZ-BUILD-PANIC")
}
func (c *F) SkipNow() {
panic("GO-FUZZ-BUILD-PANIC")
}
func (c *F) Skipf(format string, args ...any) {
panic("GO-FUZZ-BUILD-PANIC")
}
func (f *F) Skipped() bool { return false }
func (f *F) TempDir() string {
dir, err := os.MkdirTemp("", "fuzzdir-")
if err != nil {
panic(err)
}
f.T.TempDirs = append(f.T.TempDirs, dir)
return dir
}

View File

@ -0,0 +1,129 @@
package testing
import (
"fmt"
"os"
"strings"
"time"
)
// T can be used to terminate the current fuzz iteration
// without terminating the whole fuzz run. To do so, simply
// panic with the text "GO-FUZZ-BUILD-PANIC" and the fuzzer
// will recover.
type T struct {
TempDirs []string
}
func NewT() *T {
tempDirs := make([]string, 0)
return &T{TempDirs: tempDirs}
}
func unsupportedApi(name string) string {
plsOpenIss := "Please open an issue https://github.com/AdamKorcz/go-118-fuzz-build if you need this feature."
var b strings.Builder
b.WriteString(fmt.Sprintf("%s is not supported when fuzzing in libFuzzer mode\n.", name))
b.WriteString(plsOpenIss)
return b.String()
}
func (t *T) Cleanup(f func()) {
f()
}
func (t *T) Deadline() (deadline time.Time, ok bool) {
panic(unsupportedApi("t.Deadline()"))
}
func (t *T) Error(args ...any) {
fmt.Println(args...)
panic("error")
}
func (t *T) Errorf(format string, args ...any) {
fmt.Printf(format+"\n", args...)
panic("errorf")
}
func (t *T) Fail() {
panic("Called T.Fail()")
}
func (t *T) FailNow() {
panic("Called T.Fail()")
panic(unsupportedApi("t.FailNow()"))
}
func (t *T) Failed() bool {
panic(unsupportedApi("t.Failed()"))
}
func (t *T) Fatal(args ...any) {
fmt.Println(args...)
panic("fatal")
}
func (t *T) Fatalf(format string, args ...any) {
fmt.Printf(format+"\n", args...)
panic("fatal")
}
func (t *T) Helper() {
// We can't support it, but it also just impacts how failures are reported, so we can ignore it
}
func (t *T) Log(args ...any) {
fmt.Println(args...)
}
func (t *T) Logf(format string, args ...any) {
fmt.Println(format)
fmt.Println(args...)
}
func (t *T) Name() string {
return "libFuzzer"
}
func (t *T) Parallel() {
panic(unsupportedApi("t.Parallel()"))
}
func (t *T) Run(name string, f func(t *T)) bool {
panic(unsupportedApi("t.Run()"))
}
func (t *T) Setenv(key, value string) {
}
func (t *T) Skip(args ...any) {
panic("GO-FUZZ-BUILD-PANIC")
}
func (t *T) SkipNow() {
panic("GO-FUZZ-BUILD-PANIC")
}
// Is not really supported. We just skip instead
// of printing any message. A log message can be
// added if need be.
func (t *T) Skipf(format string, args ...any) {
panic("GO-FUZZ-BUILD-PANIC")
}
func (t *T) Skipped() bool {
panic(unsupportedApi("t.Skipped()"))
}
func (t *T) TempDir() string {
dir, err := os.MkdirTemp("", "fuzzdir-")
if err != nil {
panic(err)
}
t.TempDirs = append(t.TempDirs, dir)
return dir
}
func (t *T) CleanupTempDirs() {
if len(t.TempDirs) > 0 {
for _, tempDir := range t.TempDirs {
os.RemoveAll(tempDir)
}
}
}

View File

@ -0,0 +1,42 @@
package testing
import (
"testing"
)
func AllocsPerRun(runs int, f func()) (avg float64) {
panic(unsupportedApi("testing.AllocsPerRun"))
}
func CoverMode() string {
panic(unsupportedApi("testing.CoverMode"))
}
func Coverage() float64 {
panic(unsupportedApi("testing.Coverage"))
}
func Init() {
panic(unsupportedApi("testing.Init"))
}
func RegisterCover(c testing.Cover) {
panic(unsupportedApi("testing.RegisterCover"))
}
func RunExamples(matchString func(pat, str string) (bool, error), examples []testing.InternalExample) (ok bool) {
panic(unsupportedApi("testing.RunExamples"))
}
func RunTests(matchString func(pat, str string) (bool, error), tests []testing.InternalTest) (ok bool) {
panic(unsupportedApi("testing.RunTests"))
}
func Short() bool {
return false
}
func Verbose() bool {
panic(unsupportedApi("testing.Verbose"))
}
type M struct {}
func (m *M) Run() (code int) {
panic("testing.M is not support in libFuzzer Mode")
}

1
vendor/github.com/Microsoft/go-winio/.gitattributes generated vendored Normal file
View File

@ -0,0 +1 @@
* text=auto eol=lf

View File

@ -1 +1,10 @@
.vscode/
*.exe
# testing
testdata
# go workspaces
go.work
go.work.sum

147
vendor/github.com/Microsoft/go-winio/.golangci.yml generated vendored Normal file
View File

@ -0,0 +1,147 @@
linters:
enable:
# style
- containedctx # struct contains a context
- dupl # duplicate code
- errname # erorrs are named correctly
- nolintlint # "//nolint" directives are properly explained
- revive # golint replacement
- unconvert # unnecessary conversions
- wastedassign
# bugs, performance, unused, etc ...
- contextcheck # function uses a non-inherited context
- errorlint # errors not wrapped for 1.13
- exhaustive # check exhaustiveness of enum switch statements
- gofmt # files are gofmt'ed
- gosec # security
- nilerr # returns nil even with non-nil error
- thelper # test helpers without t.Helper()
- unparam # unused function params
issues:
exclude-dirs:
- pkg/etw/sample
exclude-rules:
# err is very often shadowed in nested scopes
- linters:
- govet
text: '^shadow: declaration of "err" shadows declaration'
# ignore long lines for skip autogen directives
- linters:
- revive
text: "^line-length-limit: "
source: "^//(go:generate|sys) "
#TODO: remove after upgrading to go1.18
# ignore comment spacing for nolint and sys directives
- linters:
- revive
text: "^comment-spacings: no space between comment delimiter and comment text"
source: "//(cspell:|nolint:|sys |todo)"
# not on go 1.18 yet, so no any
- linters:
- revive
text: "^use-any: since GO 1.18 'interface{}' can be replaced by 'any'"
# allow unjustified ignores of error checks in defer statements
- linters:
- nolintlint
text: "^directive `//nolint:errcheck` should provide explanation"
source: '^\s*defer '
# allow unjustified ignores of error lints for io.EOF
- linters:
- nolintlint
text: "^directive `//nolint:errorlint` should provide explanation"
source: '[=|!]= io.EOF'
linters-settings:
exhaustive:
default-signifies-exhaustive: true
govet:
enable-all: true
disable:
# struct order is often for Win32 compat
# also, ignore pointer bytes/GC issues for now until performance becomes an issue
- fieldalignment
nolintlint:
require-explanation: true
require-specific: true
revive:
# revive is more configurable than static check, so likely the preferred alternative to static-check
# (once the perf issue is solved: https://github.com/golangci/golangci-lint/issues/2997)
enable-all-rules:
true
# https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md
rules:
# rules with required arguments
- name: argument-limit
disabled: true
- name: banned-characters
disabled: true
- name: cognitive-complexity
disabled: true
- name: cyclomatic
disabled: true
- name: file-header
disabled: true
- name: function-length
disabled: true
- name: function-result-limit
disabled: true
- name: max-public-structs
disabled: true
# geneally annoying rules
- name: add-constant # complains about any and all strings and integers
disabled: true
- name: confusing-naming # we frequently use "Foo()" and "foo()" together
disabled: true
- name: flag-parameter # excessive, and a common idiom we use
disabled: true
- name: unhandled-error # warns over common fmt.Print* and io.Close; rely on errcheck instead
disabled: true
# general config
- name: line-length-limit
arguments:
- 140
- name: var-naming
arguments:
- []
- - CID
- CRI
- CTRD
- DACL
- DLL
- DOS
- ETW
- FSCTL
- GCS
- GMSA
- HCS
- HV
- IO
- LCOW
- LDAP
- LPAC
- LTSC
- MMIO
- NT
- OCI
- PMEM
- PWSH
- RX
- SACl
- SID
- SMB
- TX
- VHD
- VHDX
- VMID
- VPCI
- WCOW
- WIM

View File

@ -13,16 +13,60 @@ Please see the LICENSE file for licensing information.
## Contributing
This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA)
declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit https://cla.microsoft.com.
This project welcomes contributions and suggestions.
Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that
you have the right to, and actually do, grant us the rights to use your contribution.
For details, visit [Microsoft CLA](https://cla.microsoft.com).
When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR
appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repos using our CLA.
When you submit a pull request, a CLA-bot will automatically determine whether you need to
provide a CLA and decorate the PR appropriately (e.g., label, comment).
Simply follow the instructions provided by the bot.
You will only need to do this once across all repos using our CLA.
We also require that contributors sign their commits using git commit -s or git commit --signoff to certify they either authored the work themselves
or otherwise have permission to use it in this project. Please see https://developercertificate.org/ for more info, as well as to make sure that you can
attest to the rules listed. Our CI uses the DCO Github app to ensure that all commits in a given PR are signed-off.
Additionally, the pull request pipeline requires the following steps to be performed before
mergining.
### Code Sign-Off
We require that contributors sign their commits using [`git commit --signoff`][git-commit-s]
to certify they either authored the work themselves or otherwise have permission to use it in this project.
A range of commits can be signed off using [`git rebase --signoff`][git-rebase-s].
Please see [the developer certificate](https://developercertificate.org) for more info,
as well as to make sure that you can attest to the rules listed.
Our CI uses the DCO Github app to ensure that all commits in a given PR are signed-off.
### Linting
Code must pass a linting stage, which uses [`golangci-lint`][lint].
The linting settings are stored in [`.golangci.yaml`](./.golangci.yaml), and can be run
automatically with VSCode by adding the following to your workspace or folder settings:
```json
"go.lintTool": "golangci-lint",
"go.lintOnSave": "package",
```
Additional editor [integrations options are also available][lint-ide].
Alternatively, `golangci-lint` can be [installed locally][lint-install] and run from the repo root:
```shell
# use . or specify a path to only lint a package
# to show all lint errors, use flags "--max-issues-per-linter=0 --max-same-issues=0"
> golangci-lint run ./...
```
### Go Generate
The pipeline checks that auto-generated code, via `go generate`, are up to date.
This can be done for the entire repo:
```shell
> go generate ./...
```
## Code of Conduct
@ -30,8 +74,16 @@ This project has adopted the [Microsoft Open Source Code of Conduct](https://ope
For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or
contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments.
## Special Thanks
Thanks to natefinch for the inspiration for this library. See https://github.com/natefinch/npipe
for another named pipe implementation.
Thanks to [natefinch][natefinch] for the inspiration for this library.
See [npipe](https://github.com/natefinch/npipe) for another named pipe implementation.
[lint]: https://golangci-lint.run/
[lint-ide]: https://golangci-lint.run/usage/integrations/#editor-integration
[lint-install]: https://golangci-lint.run/usage/install/#local-installation
[git-commit-s]: https://git-scm.com/docs/git-commit#Documentation/git-commit.txt--s
[git-rebase-s]: https://git-scm.com/docs/git-rebase#Documentation/git-rebase.txt---signoff
[natefinch]: https://github.com/natefinch

41
vendor/github.com/Microsoft/go-winio/SECURITY.md generated vendored Normal file
View File

@ -0,0 +1,41 @@
<!-- BEGIN MICROSOFT SECURITY.MD V0.0.7 BLOCK -->
## Security
Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below.
## Reporting Security Issues
**Please do not report security vulnerabilities through public GitHub issues.**
Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report).
If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey).
You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc).
Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
* Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
* Full paths of source file(s) related to the manifestation of the issue
* The location of the affected source code (tag/branch/commit or direct URL)
* Any special configuration required to reproduce the issue
* Step-by-step instructions to reproduce the issue
* Proof-of-concept or exploit code (if possible)
* Impact of the issue, including how an attacker might exploit the issue
This information will help us triage your report more quickly.
If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs.
## Preferred Languages
We prefer all communications to be in English.
## Policy
Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd).
<!-- END MICROSOFT SECURITY.MD BLOCK -->

View File

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@ -7,15 +8,16 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"runtime"
"syscall"
"unicode/utf16"
"github.com/Microsoft/go-winio/internal/fs"
"golang.org/x/sys/windows"
)
//sys backupRead(h syscall.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupRead
//sys backupWrite(h syscall.Handle, b []byte, bytesWritten *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupWrite
//sys backupRead(h windows.Handle, b []byte, bytesRead *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupRead
//sys backupWrite(h windows.Handle, b []byte, bytesWritten *uint32, abort bool, processSecurity bool, context *uintptr) (err error) = BackupWrite
const (
BackupData = uint32(iota + 1)
@ -24,7 +26,7 @@ const (
BackupAlternateData
BackupLink
BackupPropertyData
BackupObjectId
BackupObjectId //revive:disable-line:var-naming ID, not Id
BackupReparseData
BackupSparseBlock
BackupTxfsData
@ -34,14 +36,16 @@ const (
StreamSparseAttributes = uint32(8)
)
//nolint:revive // var-naming: ALL_CAPS
const (
WRITE_DAC = 0x40000
WRITE_OWNER = 0x80000
ACCESS_SYSTEM_SECURITY = 0x1000000
WRITE_DAC = windows.WRITE_DAC
WRITE_OWNER = windows.WRITE_OWNER
ACCESS_SYSTEM_SECURITY = windows.ACCESS_SYSTEM_SECURITY
)
// BackupHeader represents a backup stream of a file.
type BackupHeader struct {
//revive:disable-next-line:var-naming ID, not Id
Id uint32 // The backup stream ID
Attributes uint32 // Stream attributes
Size int64 // The size of the stream in bytes
@ -49,8 +53,8 @@ type BackupHeader struct {
Offset int64 // The offset of the stream in the file (for BackupSparseBlock only).
}
type win32StreamId struct {
StreamId uint32
type win32StreamID struct {
StreamID uint32
Attributes uint32
Size uint64
NameSize uint32
@ -71,7 +75,7 @@ func NewBackupStreamReader(r io.Reader) *BackupStreamReader {
// Next returns the next backup stream and prepares for calls to Read(). It skips the remainder of the current stream if
// it was not completely read.
func (r *BackupStreamReader) Next() (*BackupHeader, error) {
if r.bytesLeft > 0 {
if r.bytesLeft > 0 { //nolint:nestif // todo: flatten this
if s, ok := r.r.(io.Seeker); ok {
// Make sure Seek on io.SeekCurrent sometimes succeeds
// before trying the actual seek.
@ -82,16 +86,16 @@ func (r *BackupStreamReader) Next() (*BackupHeader, error) {
r.bytesLeft = 0
}
}
if _, err := io.Copy(ioutil.Discard, r); err != nil {
if _, err := io.Copy(io.Discard, r); err != nil {
return nil, err
}
}
var wsi win32StreamId
var wsi win32StreamID
if err := binary.Read(r.r, binary.LittleEndian, &wsi); err != nil {
return nil, err
}
hdr := &BackupHeader{
Id: wsi.StreamId,
Id: wsi.StreamID,
Attributes: wsi.Attributes,
Size: int64(wsi.Size),
}
@ -100,9 +104,9 @@ func (r *BackupStreamReader) Next() (*BackupHeader, error) {
if err := binary.Read(r.r, binary.LittleEndian, name); err != nil {
return nil, err
}
hdr.Name = syscall.UTF16ToString(name)
hdr.Name = windows.UTF16ToString(name)
}
if wsi.StreamId == BackupSparseBlock {
if wsi.StreamID == BackupSparseBlock {
if err := binary.Read(r.r, binary.LittleEndian, &hdr.Offset); err != nil {
return nil, err
}
@ -147,8 +151,8 @@ func (w *BackupStreamWriter) WriteHeader(hdr *BackupHeader) error {
return fmt.Errorf("missing %d bytes", w.bytesLeft)
}
name := utf16.Encode([]rune(hdr.Name))
wsi := win32StreamId{
StreamId: hdr.Id,
wsi := win32StreamID{
StreamID: hdr.Id,
Attributes: hdr.Attributes,
Size: uint64(hdr.Size),
NameSize: uint32(len(name) * 2),
@ -201,9 +205,9 @@ func NewBackupFileReader(f *os.File, includeSecurity bool) *BackupFileReader {
// Read reads a backup stream from the file by calling the Win32 API BackupRead().
func (r *BackupFileReader) Read(b []byte) (int, error) {
var bytesRead uint32
err := backupRead(syscall.Handle(r.f.Fd()), b, &bytesRead, false, r.includeSecurity, &r.ctx)
err := backupRead(windows.Handle(r.f.Fd()), b, &bytesRead, false, r.includeSecurity, &r.ctx)
if err != nil {
return 0, &os.PathError{"BackupRead", r.f.Name(), err}
return 0, &os.PathError{Op: "BackupRead", Path: r.f.Name(), Err: err}
}
runtime.KeepAlive(r.f)
if bytesRead == 0 {
@ -216,7 +220,7 @@ func (r *BackupFileReader) Read(b []byte) (int, error) {
// the underlying file.
func (r *BackupFileReader) Close() error {
if r.ctx != 0 {
backupRead(syscall.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx)
_ = backupRead(windows.Handle(r.f.Fd()), nil, nil, true, false, &r.ctx)
runtime.KeepAlive(r.f)
r.ctx = 0
}
@ -240,9 +244,9 @@ func NewBackupFileWriter(f *os.File, includeSecurity bool) *BackupFileWriter {
// Write restores a portion of the file using the provided backup stream.
func (w *BackupFileWriter) Write(b []byte) (int, error) {
var bytesWritten uint32
err := backupWrite(syscall.Handle(w.f.Fd()), b, &bytesWritten, false, w.includeSecurity, &w.ctx)
err := backupWrite(windows.Handle(w.f.Fd()), b, &bytesWritten, false, w.includeSecurity, &w.ctx)
if err != nil {
return 0, &os.PathError{"BackupWrite", w.f.Name(), err}
return 0, &os.PathError{Op: "BackupWrite", Path: w.f.Name(), Err: err}
}
runtime.KeepAlive(w.f)
if int(bytesWritten) != len(b) {
@ -255,7 +259,7 @@ func (w *BackupFileWriter) Write(b []byte) (int, error) {
// close the underlying file.
func (w *BackupFileWriter) Close() error {
if w.ctx != 0 {
backupWrite(syscall.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx)
_ = backupWrite(windows.Handle(w.f.Fd()), nil, nil, true, false, &w.ctx)
runtime.KeepAlive(w.f)
w.ctx = 0
}
@ -267,11 +271,14 @@ func (w *BackupFileWriter) Close() error {
//
// If the file opened was a directory, it cannot be used with Readdir().
func OpenForBackup(path string, access uint32, share uint32, createmode uint32) (*os.File, error) {
winPath, err := syscall.UTF16FromString(path)
if err != nil {
return nil, err
}
h, err := syscall.CreateFile(&winPath[0], access, share, nil, createmode, syscall.FILE_FLAG_BACKUP_SEMANTICS|syscall.FILE_FLAG_OPEN_REPARSE_POINT, 0)
h, err := fs.CreateFile(path,
fs.AccessMask(access),
fs.FileShareMode(share),
nil,
fs.FileCreationDisposition(createmode),
fs.FILE_FLAG_BACKUP_SEMANTICS|fs.FILE_FLAG_OPEN_REPARSE_POINT,
0,
)
if err != nil {
err = &os.PathError{Op: "open", Path: path, Err: err}
return nil, err

View File

@ -1,4 +1,3 @@
// +build !windows
// This file only exists to allow go get on non-Windows platforms.
package backuptar

View File

@ -1,3 +1,5 @@
//go:build windows
package backuptar
import (

View File

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package backuptar
@ -7,28 +8,27 @@ import (
"encoding/base64"
"fmt"
"io"
"io/ioutil"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
"github.com/Microsoft/go-winio"
"golang.org/x/sys/windows"
)
//nolint:deadcode,varcheck // keep unused constants for potential future use
const (
c_ISUID = 04000 // Set uid
c_ISGID = 02000 // Set gid
c_ISVTX = 01000 // Save text (sticky bit)
c_ISDIR = 040000 // Directory
c_ISFIFO = 010000 // FIFO
c_ISREG = 0100000 // Regular file
c_ISLNK = 0120000 // Symbolic link
c_ISBLK = 060000 // Block special file
c_ISCHR = 020000 // Character special file
c_ISSOCK = 0140000 // Socket
cISUID = 0004000 // Set uid
cISGID = 0002000 // Set gid
cISVTX = 0001000 // Save text (sticky bit)
cISDIR = 0040000 // Directory
cISFIFO = 0010000 // FIFO
cISREG = 0100000 // Regular file
cISLNK = 0120000 // Symbolic link
cISBLK = 0060000 // Block special file
cISCHR = 0020000 // Character special file
cISSOCK = 0140000 // Socket
)
const (
@ -44,7 +44,7 @@ const (
// zeroReader is an io.Reader that always returns 0s.
type zeroReader struct{}
func (zr zeroReader) Read(b []byte) (int, error) {
func (zeroReader) Read(b []byte) (int, error) {
for i := range b {
b[i] = 0
}
@ -55,7 +55,7 @@ func copySparse(t *tar.Writer, br *winio.BackupStreamReader) error {
curOffset := int64(0)
for {
bhdr, err := br.Next()
if err == io.EOF {
if err == io.EOF { //nolint:errorlint
err = io.ErrUnexpectedEOF
}
if err != nil {
@ -71,8 +71,8 @@ func copySparse(t *tar.Writer, br *winio.BackupStreamReader) error {
}
// archive/tar does not support writing sparse files
// so just write zeroes to catch up to the current offset.
if _, err := io.CopyN(t, zeroReader{}, bhdr.Offset-curOffset); err != nil {
return fmt.Errorf("seek to offset %d: %s", bhdr.Offset, err)
if _, err = io.CopyN(t, zeroReader{}, bhdr.Offset-curOffset); err != nil {
return fmt.Errorf("seek to offset %d: %w", bhdr.Offset, err)
}
if bhdr.Size == 0 {
// A sparse block with size = 0 is used to mark the end of the sparse blocks.
@ -105,25 +105,85 @@ func BasicInfoHeader(name string, size int64, fileInfo *winio.FileBasicInfo) *ta
hdr.PAXRecords[hdrFileAttributes] = fmt.Sprintf("%d", fileInfo.FileAttributes)
hdr.PAXRecords[hdrCreationTime] = formatPAXTime(time.Unix(0, fileInfo.CreationTime.Nanoseconds()))
if (fileInfo.FileAttributes & syscall.FILE_ATTRIBUTE_DIRECTORY) != 0 {
hdr.Mode |= c_ISDIR
if (fileInfo.FileAttributes & windows.FILE_ATTRIBUTE_DIRECTORY) != 0 {
hdr.Mode |= cISDIR
hdr.Size = 0
hdr.Typeflag = tar.TypeDir
}
return hdr
}
// SecurityDescriptorFromTarHeader reads the SDDL associated with the header of the current file
// from the tar header and returns the security descriptor into a byte slice.
func SecurityDescriptorFromTarHeader(hdr *tar.Header) ([]byte, error) {
if sdraw, ok := hdr.PAXRecords[hdrRawSecurityDescriptor]; ok {
sd, err := base64.StdEncoding.DecodeString(sdraw)
if err != nil {
// Not returning sd as-is in the error-case, as base64.DecodeString
// may return partially decoded data (not nil or empty slice) in case
// of a failure: https://github.com/golang/go/blob/go1.17.7/src/encoding/base64/base64.go#L382-L387
return nil, err
}
return sd, nil
}
// Maintaining old SDDL-based behavior for backward compatibility. All new
// tar headers written by this library will have raw binary for the security
// descriptor.
if sddl, ok := hdr.PAXRecords[hdrSecurityDescriptor]; ok {
return winio.SddlToSecurityDescriptor(sddl)
}
return nil, nil
}
// ExtendedAttributesFromTarHeader reads the EAs associated with the header of the
// current file from the tar header and returns it as a byte slice.
func ExtendedAttributesFromTarHeader(hdr *tar.Header) ([]byte, error) {
var eas []winio.ExtendedAttribute //nolint:prealloc // len(eas) <= len(hdr.PAXRecords); prealloc is wasteful
for k, v := range hdr.PAXRecords {
if !strings.HasPrefix(k, hdrEaPrefix) {
continue
}
data, err := base64.StdEncoding.DecodeString(v)
if err != nil {
return nil, err
}
eas = append(eas, winio.ExtendedAttribute{
Name: k[len(hdrEaPrefix):],
Value: data,
})
}
var eaData []byte
var err error
if len(eas) != 0 {
eaData, err = winio.EncodeExtendedAttributes(eas)
if err != nil {
return nil, err
}
}
return eaData, nil
}
// EncodeReparsePointFromTarHeader reads the ReparsePoint structure from the tar header
// and encodes it into a byte slice. The file for which this function is called must be a
// symlink.
func EncodeReparsePointFromTarHeader(hdr *tar.Header) []byte {
_, isMountPoint := hdr.PAXRecords[hdrMountPoint]
rp := winio.ReparsePoint{
Target: filepath.FromSlash(hdr.Linkname),
IsMountPoint: isMountPoint,
}
return winio.EncodeReparsePoint(&rp)
}
// WriteTarFileFromBackupStream writes a file to a tar writer using data from a Win32 backup stream.
//
// This encodes Win32 metadata as tar pax vendor extensions starting with MSWINDOWS.
//
// The additional Win32 metadata is:
//
// MSWINDOWS.fileattr: The Win32 file attributes, as a decimal value
//
// MSWINDOWS.rawsd: The Win32 security descriptor, in raw binary format
//
// MSWINDOWS.mountpoint: If present, this is a mount point and not a symlink, even though the type is '2' (symlink)
// - MSWINDOWS.fileattr: The Win32 file attributes, as a decimal value
// - MSWINDOWS.rawsd: The Win32 security descriptor, in raw binary format
// - MSWINDOWS.mountpoint: If present, this is a mount point and not a symlink, even though the type is '2' (symlink)
func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size int64, fileInfo *winio.FileBasicInfo) error {
name = filepath.ToSlash(name)
hdr := BasicInfoHeader(name, size, fileInfo)
@ -146,7 +206,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
var dataHdr *winio.BackupHeader
for dataHdr == nil {
bhdr, err := br.Next()
if err == io.EOF {
if err == io.EOF { //nolint:errorlint
break
}
if err != nil {
@ -154,21 +214,21 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
}
switch bhdr.Id {
case winio.BackupData:
hdr.Mode |= c_ISREG
hdr.Mode |= cISREG
if !readTwice {
dataHdr = bhdr
}
case winio.BackupSecurity:
sd, err := ioutil.ReadAll(br)
sd, err := io.ReadAll(br)
if err != nil {
return err
}
hdr.PAXRecords[hdrRawSecurityDescriptor] = base64.StdEncoding.EncodeToString(sd)
case winio.BackupReparseData:
hdr.Mode |= c_ISLNK
hdr.Mode |= cISLNK
hdr.Typeflag = tar.TypeSymlink
reparseBuffer, err := ioutil.ReadAll(br)
reparseBuffer, _ := io.ReadAll(br)
rp, err := winio.DecodeReparsePoint(reparseBuffer)
if err != nil {
return err
@ -179,7 +239,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
hdr.Linkname = rp.Target
case winio.BackupEaData:
eab, err := ioutil.ReadAll(br)
eab, err := io.ReadAll(br)
if err != nil {
return err
}
@ -213,7 +273,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
}
for dataHdr == nil {
bhdr, err := br.Next()
if err == io.EOF {
if err == io.EOF { //nolint:errorlint
break
}
if err != nil {
@ -248,7 +308,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
// range of the file containing the range contents. Finally there is a sparse block stream with
// size = 0 and offset = <file size>.
if dataHdr != nil {
if dataHdr != nil { //nolint:nestif // todo: reduce nesting complexity
// A data stream was found. Copy the data.
// We assume that we will either have a data stream size > 0 XOR have sparse block streams.
if dataHdr.Size > 0 || (dataHdr.Attributes&winio.StreamSparseAttributes) == 0 {
@ -256,13 +316,13 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
return fmt.Errorf("%s: mismatch between file size %d and header size %d", name, size, dataHdr.Size)
}
if _, err = io.Copy(t, br); err != nil {
return fmt.Errorf("%s: copying contents from data stream: %s", name, err)
return fmt.Errorf("%s: copying contents from data stream: %w", name, err)
}
} else if size > 0 {
// As of a recent OS change, BackupRead now returns a data stream for empty sparse files.
// These files have no sparse block streams, so skip the copySparse call if file size = 0.
if err = copySparse(t, br); err != nil {
return fmt.Errorf("%s: copying contents from sparse block stream: %s", name, err)
return fmt.Errorf("%s: copying contents from sparse block stream: %w", name, err)
}
}
}
@ -272,7 +332,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
// been written. In practice, this means that we don't get EA or TXF metadata.
for {
bhdr, err := br.Next()
if err == io.EOF {
if err == io.EOF { //nolint:errorlint
break
}
if err != nil {
@ -280,11 +340,12 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
}
switch bhdr.Id {
case winio.BackupAlternateData:
altName := bhdr.Name
if strings.HasSuffix(altName, ":$DATA") {
altName = altName[:len(altName)-len(":$DATA")]
if (bhdr.Attributes & winio.StreamSparseAttributes) != 0 {
// Unsupported for now, since the size of the alternate stream is not present
// in the backup stream until after the data has been read.
return fmt.Errorf("%s: tar of sparse alternate data streams is unsupported", name)
}
if (bhdr.Attributes & winio.StreamSparseAttributes) == 0 {
altName := strings.TrimSuffix(bhdr.Name, ":$DATA")
hdr = &tar.Header{
Format: hdr.Format,
Name: name + altName,
@ -303,12 +364,6 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
if err != nil {
return err
}
} else {
// Unsupported for now, since the size of the alternate stream is not present
// in the backup stream until after the data has been read.
return fmt.Errorf("%s: tar of sparse alternate data streams is unsupported", name)
}
case winio.BackupEaData, winio.BackupLink, winio.BackupPropertyData, winio.BackupObjectId, winio.BackupTxfsData:
// ignore these streams
default:
@ -322,7 +377,7 @@ func WriteTarFileFromBackupStream(t *tar.Writer, r io.Reader, name string, size
// WriteTarFileFromBackupStream.
func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *winio.FileBasicInfo, err error) {
name = hdr.Name
if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA {
if hdr.Typeflag == tar.TypeReg {
size = hdr.Size
}
fileInfo = &winio.FileBasicInfo{
@ -340,7 +395,7 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win
fileInfo.FileAttributes = uint32(attr)
} else {
if hdr.Typeflag == tar.TypeDir {
fileInfo.FileAttributes |= syscall.FILE_ATTRIBUTE_DIRECTORY
fileInfo.FileAttributes |= windows.FILE_ATTRIBUTE_DIRECTORY
}
}
if creationTimeStr, ok := hdr.PAXRecords[hdrCreationTime]; ok {
@ -350,7 +405,7 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win
}
fileInfo.CreationTime = windows.NsecToFiletime(creationTime.UnixNano())
}
return
return name, size, fileInfo, err
}
// WriteBackupStreamFromTarFile writes a Win32 backup stream from the current tar file. Since this function may process multiple
@ -358,22 +413,11 @@ func FileInfoFromHeader(hdr *tar.Header) (name string, size int64, fileInfo *win
// tar file that was not processed, or io.EOF is there are no more.
func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (*tar.Header, error) {
bw := winio.NewBackupStreamWriter(w)
var sd []byte
var err error
// Maintaining old SDDL-based behavior for backward compatibility. All new tar headers written
// by this library will have raw binary for the security descriptor.
if sddl, ok := hdr.PAXRecords[hdrSecurityDescriptor]; ok {
sd, err = winio.SddlToSecurityDescriptor(sddl)
sd, err := SecurityDescriptorFromTarHeader(hdr)
if err != nil {
return nil, err
}
}
if sdraw, ok := hdr.PAXRecords[hdrRawSecurityDescriptor]; ok {
sd, err = base64.StdEncoding.DecodeString(sdraw)
if err != nil {
return nil, err
}
}
if len(sd) != 0 {
bhdr := winio.BackupHeader{
Id: winio.BackupSecurity,
@ -388,25 +432,12 @@ func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (
return nil, err
}
}
var eas []winio.ExtendedAttribute
for k, v := range hdr.PAXRecords {
if !strings.HasPrefix(k, hdrEaPrefix) {
continue
}
data, err := base64.StdEncoding.DecodeString(v)
if err != nil {
return nil, err
}
eas = append(eas, winio.ExtendedAttribute{
Name: k[len(hdrEaPrefix):],
Value: data,
})
}
if len(eas) != 0 {
eadata, err := winio.EncodeExtendedAttributes(eas)
eadata, err := ExtendedAttributesFromTarHeader(hdr)
if err != nil {
return nil, err
}
if len(eadata) != 0 {
bhdr := winio.BackupHeader{
Id: winio.BackupEaData,
Size: int64(len(eadata)),
@ -420,13 +451,9 @@ func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (
return nil, err
}
}
if hdr.Typeflag == tar.TypeSymlink {
_, isMountPoint := hdr.PAXRecords[hdrMountPoint]
rp := winio.ReparsePoint{
Target: filepath.FromSlash(hdr.Linkname),
IsMountPoint: isMountPoint,
}
reparse := winio.EncodeReparsePoint(&rp)
reparse := EncodeReparsePointFromTarHeader(hdr)
bhdr := winio.BackupHeader{
Id: winio.BackupReparseData,
Size: int64(len(reparse)),
@ -440,7 +467,8 @@ func WriteBackupStreamFromTarFile(w io.Writer, t *tar.Reader, hdr *tar.Header) (
return nil, err
}
}
if hdr.Typeflag == tar.TypeReg || hdr.Typeflag == tar.TypeRegA {
if hdr.Typeflag == tar.TypeReg {
bhdr := winio.BackupHeader{
Id: winio.BackupData,
Size: hdr.Size,

22
vendor/github.com/Microsoft/go-winio/doc.go generated vendored Normal file
View File

@ -0,0 +1,22 @@
// This package provides utilities for efficiently performing Win32 IO operations in Go.
// Currently, this package is provides support for genreal IO and management of
// - named pipes
// - files
// - [Hyper-V sockets]
//
// This code is similar to Go's [net] package, and uses IO completion ports to avoid
// blocking IO on system threads, allowing Go to reuse the thread to schedule other goroutines.
//
// This limits support to Windows Vista and newer operating systems.
//
// Additionally, this package provides support for:
// - creating and managing GUIDs
// - writing to [ETW]
// - opening and manageing VHDs
// - parsing [Windows Image files]
// - auto-generating Win32 API code
//
// [Hyper-V sockets]: https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/make-integration-service
// [ETW]: https://docs.microsoft.com/en-us/windows-hardware/drivers/devtest/event-tracing-for-windows--etw-
// [Windows Image files]: https://docs.microsoft.com/en-us/windows-hardware/manufacture/desktop/work-with-windows-images
package winio

View File

@ -33,7 +33,7 @@ func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
err = binary.Read(bytes.NewReader(b), binary.LittleEndian, &info)
if err != nil {
err = errInvalidEaBuffer
return
return ea, nb, err
}
nameOffset := fileFullEaInformationSize
@ -43,7 +43,7 @@ func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
nextOffset := int(info.NextEntryOffset)
if valueLen+valueOffset > len(b) || nextOffset < 0 || nextOffset > len(b) {
err = errInvalidEaBuffer
return
return ea, nb, err
}
ea.Name = string(b[nameOffset : nameOffset+nameLen])
@ -52,7 +52,7 @@ func parseEa(b []byte) (ea ExtendedAttribute, nb []byte, err error) {
if info.NextEntryOffset != 0 {
nb = b[info.NextEntryOffset:]
}
return
return ea, nb, err
}
// DecodeExtendedAttributes decodes a list of EAs from a FILE_FULL_EA_INFORMATION
@ -67,7 +67,7 @@ func DecodeExtendedAttributes(b []byte) (eas []ExtendedAttribute, err error) {
eas = append(eas, ea)
b = nb
}
return
return eas, err
}
func writeEa(buf *bytes.Buffer, ea *ExtendedAttribute, last bool) error {

View File

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@ -10,31 +11,15 @@ import (
"sync/atomic"
"syscall"
"time"
"golang.org/x/sys/windows"
)
//sys cancelIoEx(file syscall.Handle, o *syscall.Overlapped) (err error) = CancelIoEx
//sys createIoCompletionPort(file syscall.Handle, port syscall.Handle, key uintptr, threadCount uint32) (newport syscall.Handle, err error) = CreateIoCompletionPort
//sys getQueuedCompletionStatus(port syscall.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) = GetQueuedCompletionStatus
//sys setFileCompletionNotificationModes(h syscall.Handle, flags uint8) (err error) = SetFileCompletionNotificationModes
//sys wsaGetOverlappedResult(h syscall.Handle, o *syscall.Overlapped, bytes *uint32, wait bool, flags *uint32) (err error) = ws2_32.WSAGetOverlappedResult
type atomicBool int32
func (b *atomicBool) isSet() bool { return atomic.LoadInt32((*int32)(b)) != 0 }
func (b *atomicBool) setFalse() { atomic.StoreInt32((*int32)(b), 0) }
func (b *atomicBool) setTrue() { atomic.StoreInt32((*int32)(b), 1) }
func (b *atomicBool) swap(new bool) bool {
var newInt int32
if new {
newInt = 1
}
return atomic.SwapInt32((*int32)(b), newInt) == 1
}
const (
cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS = 1
cFILE_SKIP_SET_EVENT_ON_HANDLE = 2
)
//sys cancelIoEx(file windows.Handle, o *windows.Overlapped) (err error) = CancelIoEx
//sys createIoCompletionPort(file windows.Handle, port windows.Handle, key uintptr, threadCount uint32) (newport windows.Handle, err error) = CreateIoCompletionPort
//sys getQueuedCompletionStatus(port windows.Handle, bytes *uint32, key *uintptr, o **ioOperation, timeout uint32) (err error) = GetQueuedCompletionStatus
//sys setFileCompletionNotificationModes(h windows.Handle, flags uint8) (err error) = SetFileCompletionNotificationModes
//sys wsaGetOverlappedResult(h windows.Handle, o *windows.Overlapped, bytes *uint32, wait bool, flags *uint32) (err error) = ws2_32.WSAGetOverlappedResult
var (
ErrFileClosed = errors.New("file has already been closed")
@ -43,29 +28,29 @@ var (
type timeoutError struct{}
func (e *timeoutError) Error() string { return "i/o timeout" }
func (e *timeoutError) Timeout() bool { return true }
func (e *timeoutError) Temporary() bool { return true }
func (*timeoutError) Error() string { return "i/o timeout" }
func (*timeoutError) Timeout() bool { return true }
func (*timeoutError) Temporary() bool { return true }
type timeoutChan chan struct{}
var ioInitOnce sync.Once
var ioCompletionPort syscall.Handle
var ioCompletionPort windows.Handle
// ioResult contains the result of an asynchronous IO operation
// ioResult contains the result of an asynchronous IO operation.
type ioResult struct {
bytes uint32
err error
}
// ioOperation represents an outstanding asynchronous Win32 IO
// ioOperation represents an outstanding asynchronous Win32 IO.
type ioOperation struct {
o syscall.Overlapped
o windows.Overlapped
ch chan ioResult
}
func initIo() {
h, err := createIoCompletionPort(syscall.InvalidHandle, 0, 0, 0xffffffff)
func initIO() {
h, err := createIoCompletionPort(windows.InvalidHandle, 0, 0, 0xffffffff)
if err != nil {
panic(err)
}
@ -76,10 +61,10 @@ func initIo() {
// win32File implements Reader, Writer, and Closer on a Win32 handle without blocking in a syscall.
// It takes ownership of this handle and will close it if it is garbage collected.
type win32File struct {
handle syscall.Handle
handle windows.Handle
wg sync.WaitGroup
wgLock sync.RWMutex
closing atomicBool
closing atomic.Bool
socket bool
readDeadline deadlineHandler
writeDeadline deadlineHandler
@ -90,18 +75,18 @@ type deadlineHandler struct {
channel timeoutChan
channelLock sync.RWMutex
timer *time.Timer
timedout atomicBool
timedout atomic.Bool
}
// makeWin32File makes a new win32File from an existing file handle
func makeWin32File(h syscall.Handle) (*win32File, error) {
// makeWin32File makes a new win32File from an existing file handle.
func makeWin32File(h windows.Handle) (*win32File, error) {
f := &win32File{handle: h}
ioInitOnce.Do(initIo)
ioInitOnce.Do(initIO)
_, err := createIoCompletionPort(h, ioCompletionPort, 0, 0xffffffff)
if err != nil {
return nil, err
}
err = setFileCompletionNotificationModes(h, cFILE_SKIP_COMPLETION_PORT_ON_SUCCESS|cFILE_SKIP_SET_EVENT_ON_HANDLE)
err = setFileCompletionNotificationModes(h, windows.FILE_SKIP_COMPLETION_PORT_ON_SUCCESS|windows.FILE_SKIP_SET_EVENT_ON_HANDLE)
if err != nil {
return nil, err
}
@ -110,7 +95,12 @@ func makeWin32File(h syscall.Handle) (*win32File, error) {
return f, nil
}
// Deprecated: use NewOpenFile instead.
func MakeOpenFile(h syscall.Handle) (io.ReadWriteCloser, error) {
return NewOpenFile(windows.Handle(h))
}
func NewOpenFile(h windows.Handle) (io.ReadWriteCloser, error) {
// If we return the result of makeWin32File directly, it can result in an
// interface-wrapped nil, rather than a nil interface value.
f, err := makeWin32File(h)
@ -120,17 +110,17 @@ func MakeOpenFile(h syscall.Handle) (io.ReadWriteCloser, error) {
return f, nil
}
// closeHandle closes the resources associated with a Win32 handle
// closeHandle closes the resources associated with a Win32 handle.
func (f *win32File) closeHandle() {
f.wgLock.Lock()
// Atomically set that we are closing, releasing the resources only once.
if !f.closing.swap(true) {
if !f.closing.Swap(true) {
f.wgLock.Unlock()
// cancel all IO and wait for it to complete
cancelIoEx(f.handle, nil)
_ = cancelIoEx(f.handle, nil)
f.wg.Wait()
// at this point, no new IO can start
syscall.Close(f.handle)
windows.Close(f.handle)
f.handle = 0
} else {
f.wgLock.Unlock()
@ -143,11 +133,16 @@ func (f *win32File) Close() error {
return nil
}
// prepareIo prepares for a new IO operation.
// IsClosed checks if the file has been closed.
func (f *win32File) IsClosed() bool {
return f.closing.Load()
}
// prepareIO prepares for a new IO operation.
// The caller must call f.wg.Done() when the IO is finished, prior to Close() returning.
func (f *win32File) prepareIo() (*ioOperation, error) {
func (f *win32File) prepareIO() (*ioOperation, error) {
f.wgLock.RLock()
if f.closing.isSet() {
if f.closing.Load() {
f.wgLock.RUnlock()
return nil, ErrFileClosed
}
@ -158,13 +153,13 @@ func (f *win32File) prepareIo() (*ioOperation, error) {
return c, nil
}
// ioCompletionProcessor processes completed async IOs forever
func ioCompletionProcessor(h syscall.Handle) {
// ioCompletionProcessor processes completed async IOs forever.
func ioCompletionProcessor(h windows.Handle) {
for {
var bytes uint32
var key uintptr
var op *ioOperation
err := getQueuedCompletionStatus(h, &bytes, &key, &op, syscall.INFINITE)
err := getQueuedCompletionStatus(h, &bytes, &key, &op, windows.INFINITE)
if op == nil {
panic(err)
}
@ -172,15 +167,17 @@ func ioCompletionProcessor(h syscall.Handle) {
}
}
// asyncIo processes the return value from ReadFile or WriteFile, blocking until
// todo: helsaawy - create an asyncIO version that takes a context
// asyncIO processes the return value from ReadFile or WriteFile, blocking until
// the operation has actually completed.
func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, err error) (int, error) {
if err != syscall.ERROR_IO_PENDING {
func (f *win32File) asyncIO(c *ioOperation, d *deadlineHandler, bytes uint32, err error) (int, error) {
if err != windows.ERROR_IO_PENDING { //nolint:errorlint // err is Errno
return int(bytes), err
}
if f.closing.isSet() {
cancelIoEx(f.handle, &c.o)
if f.closing.Load() {
_ = cancelIoEx(f.handle, &c.o)
}
var timeout timeoutChan
@ -194,8 +191,8 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er
select {
case r = <-c.ch:
err = r.err
if err == syscall.ERROR_OPERATION_ABORTED {
if f.closing.isSet() {
if err == windows.ERROR_OPERATION_ABORTED { //nolint:errorlint // err is Errno
if f.closing.Load() {
err = ErrFileClosed
}
} else if err != nil && f.socket {
@ -204,10 +201,10 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er
err = wsaGetOverlappedResult(f.handle, &c.o, &bytes, false, &flags)
}
case <-timeout:
cancelIoEx(f.handle, &c.o)
_ = cancelIoEx(f.handle, &c.o)
r = <-c.ch
err = r.err
if err == syscall.ERROR_OPERATION_ABORTED {
if err == windows.ERROR_OPERATION_ABORTED { //nolint:errorlint // err is Errno
err = ErrTimeout
}
}
@ -215,52 +212,52 @@ func (f *win32File) asyncIo(c *ioOperation, d *deadlineHandler, bytes uint32, er
// runtime.KeepAlive is needed, as c is passed via native
// code to ioCompletionProcessor, c must remain alive
// until the channel read is complete.
// todo: (de)allocate *ioOperation via win32 heap functions, instead of needing to KeepAlive?
runtime.KeepAlive(c)
return int(r.bytes), err
}
// Read reads from a file handle.
func (f *win32File) Read(b []byte) (int, error) {
c, err := f.prepareIo()
c, err := f.prepareIO()
if err != nil {
return 0, err
}
defer f.wg.Done()
if f.readDeadline.timedout.isSet() {
if f.readDeadline.timedout.Load() {
return 0, ErrTimeout
}
var bytes uint32
err = syscall.ReadFile(f.handle, b, &bytes, &c.o)
n, err := f.asyncIo(c, &f.readDeadline, bytes, err)
err = windows.ReadFile(f.handle, b, &bytes, &c.o)
n, err := f.asyncIO(c, &f.readDeadline, bytes, err)
runtime.KeepAlive(b)
// Handle EOF conditions.
if err == nil && n == 0 && len(b) != 0 {
return 0, io.EOF
} else if err == syscall.ERROR_BROKEN_PIPE {
} else if err == windows.ERROR_BROKEN_PIPE { //nolint:errorlint // err is Errno
return 0, io.EOF
} else {
return n, err
}
return n, err
}
// Write writes to a file handle.
func (f *win32File) Write(b []byte) (int, error) {
c, err := f.prepareIo()
c, err := f.prepareIO()
if err != nil {
return 0, err
}
defer f.wg.Done()
if f.writeDeadline.timedout.isSet() {
if f.writeDeadline.timedout.Load() {
return 0, ErrTimeout
}
var bytes uint32
err = syscall.WriteFile(f.handle, b, &bytes, &c.o)
n, err := f.asyncIo(c, &f.writeDeadline, bytes, err)
err = windows.WriteFile(f.handle, b, &bytes, &c.o)
n, err := f.asyncIO(c, &f.writeDeadline, bytes, err)
runtime.KeepAlive(b)
return n, err
}
@ -274,7 +271,7 @@ func (f *win32File) SetWriteDeadline(deadline time.Time) error {
}
func (f *win32File) Flush() error {
return syscall.FlushFileBuffers(f.handle)
return windows.FlushFileBuffers(f.handle)
}
func (f *win32File) Fd() uintptr {
@ -291,7 +288,7 @@ func (d *deadlineHandler) set(deadline time.Time) error {
}
d.timer = nil
}
d.timedout.setFalse()
d.timedout.Store(false)
select {
case <-d.channel:
@ -306,7 +303,7 @@ func (d *deadlineHandler) set(deadline time.Time) error {
}
timeoutIO := func() {
d.timedout.setTrue()
d.timedout.Store(true)
close(d.channel)
}

View File

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@ -14,22 +15,46 @@ import (
type FileBasicInfo struct {
CreationTime, LastAccessTime, LastWriteTime, ChangeTime windows.Filetime
FileAttributes uint32
pad uint32 // padding
_ uint32 // padding
}
// alignedFileBasicInfo is a FileBasicInfo, but aligned to uint64 by containing
// uint64 rather than windows.Filetime. Filetime contains two uint32s. uint64
// alignment is necessary to pass this as FILE_BASIC_INFO.
type alignedFileBasicInfo struct {
CreationTime, LastAccessTime, LastWriteTime, ChangeTime uint64
FileAttributes uint32
_ uint32 // padding
}
// GetFileBasicInfo retrieves times and attributes for a file.
func GetFileBasicInfo(f *os.File) (*FileBasicInfo, error) {
bi := &FileBasicInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
bi := &alignedFileBasicInfo{}
if err := windows.GetFileInformationByHandleEx(
windows.Handle(f.Fd()),
windows.FileBasicInfo,
(*byte)(unsafe.Pointer(bi)),
uint32(unsafe.Sizeof(*bi)),
); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
return bi, nil
// Reinterpret the alignedFileBasicInfo as a FileBasicInfo so it matches the
// public API of this module. The data may be unnecessarily aligned.
return (*FileBasicInfo)(unsafe.Pointer(bi)), nil
}
// SetFileBasicInfo sets times and attributes for a file.
func SetFileBasicInfo(f *os.File, bi *FileBasicInfo) error {
if err := windows.SetFileInformationByHandle(windows.Handle(f.Fd()), windows.FileBasicInfo, (*byte)(unsafe.Pointer(bi)), uint32(unsafe.Sizeof(*bi))); err != nil {
// Create an alignedFileBasicInfo based on a FileBasicInfo. The copy is
// suitable to pass to GetFileInformationByHandleEx.
biAligned := *(*alignedFileBasicInfo)(unsafe.Pointer(bi))
if err := windows.SetFileInformationByHandle(
windows.Handle(f.Fd()),
windows.FileBasicInfo,
(*byte)(unsafe.Pointer(&biAligned)),
uint32(unsafe.Sizeof(biAligned)),
); err != nil {
return &os.PathError{Op: "SetFileInformationByHandle", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
@ -48,7 +73,10 @@ type FileStandardInfo struct {
// GetFileStandardInfo retrieves ended information for the file.
func GetFileStandardInfo(f *os.File) (*FileStandardInfo, error) {
si := &FileStandardInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileStandardInfo, (*byte)(unsafe.Pointer(si)), uint32(unsafe.Sizeof(*si))); err != nil {
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()),
windows.FileStandardInfo,
(*byte)(unsafe.Pointer(si)),
uint32(unsafe.Sizeof(*si))); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)
@ -65,7 +93,12 @@ type FileIDInfo struct {
// GetFileID retrieves the unique (volume, file ID) pair for a file.
func GetFileID(f *os.File) (*FileIDInfo, error) {
fileID := &FileIDInfo{}
if err := windows.GetFileInformationByHandleEx(windows.Handle(f.Fd()), windows.FileIdInfo, (*byte)(unsafe.Pointer(fileID)), uint32(unsafe.Sizeof(*fileID))); err != nil {
if err := windows.GetFileInformationByHandleEx(
windows.Handle(f.Fd()),
windows.FileIdInfo,
(*byte)(unsafe.Pointer(fileID)),
uint32(unsafe.Sizeof(*fileID)),
); err != nil {
return nil, &os.PathError{Op: "GetFileInformationByHandleEx", Path: f.Name(), Err: err}
}
runtime.KeepAlive(f)

View File

@ -1,26 +1,99 @@
//go:build windows
// +build windows
package winio
import (
"context"
"errors"
"fmt"
"io"
"net"
"os"
"syscall"
"time"
"unsafe"
"golang.org/x/sys/windows"
"github.com/Microsoft/go-winio/internal/socket"
"github.com/Microsoft/go-winio/pkg/guid"
)
//sys bind(s syscall.Handle, name unsafe.Pointer, namelen int32) (err error) [failretval==socketError] = ws2_32.bind
const afHVSock = 34 // AF_HYPERV
const (
afHvSock = 34 // AF_HYPERV
// Well known Service and VM IDs
// https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/user-guide/make-integration-service#vmid-wildcards
socketError = ^uintptr(0)
)
// HvsockGUIDWildcard is the wildcard VmId for accepting connections from all partitions.
func HvsockGUIDWildcard() guid.GUID { // 00000000-0000-0000-0000-000000000000
return guid.GUID{}
}
// HvsockGUIDBroadcast is the wildcard VmId for broadcasting sends to all partitions.
func HvsockGUIDBroadcast() guid.GUID { // ffffffff-ffff-ffff-ffff-ffffffffffff
return guid.GUID{
Data1: 0xffffffff,
Data2: 0xffff,
Data3: 0xffff,
Data4: [8]uint8{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
}
}
// HvsockGUIDLoopback is the Loopback VmId for accepting connections to the same partition as the connector.
func HvsockGUIDLoopback() guid.GUID { // e0e16197-dd56-4a10-9195-5ee7a155a838
return guid.GUID{
Data1: 0xe0e16197,
Data2: 0xdd56,
Data3: 0x4a10,
Data4: [8]uint8{0x91, 0x95, 0x5e, 0xe7, 0xa1, 0x55, 0xa8, 0x38},
}
}
// HvsockGUIDSiloHost is the address of a silo's host partition:
// - The silo host of a hosted silo is the utility VM.
// - The silo host of a silo on a physical host is the physical host.
func HvsockGUIDSiloHost() guid.GUID { // 36bd0c5c-7276-4223-88ba-7d03b654c568
return guid.GUID{
Data1: 0x36bd0c5c,
Data2: 0x7276,
Data3: 0x4223,
Data4: [8]byte{0x88, 0xba, 0x7d, 0x03, 0xb6, 0x54, 0xc5, 0x68},
}
}
// HvsockGUIDChildren is the wildcard VmId for accepting connections from the connector's child partitions.
func HvsockGUIDChildren() guid.GUID { // 90db8b89-0d35-4f79-8ce9-49ea0ac8b7cd
return guid.GUID{
Data1: 0x90db8b89,
Data2: 0xd35,
Data3: 0x4f79,
Data4: [8]uint8{0x8c, 0xe9, 0x49, 0xea, 0xa, 0xc8, 0xb7, 0xcd},
}
}
// HvsockGUIDParent is the wildcard VmId for accepting connections from the connector's parent partition.
// Listening on this VmId accepts connection from:
// - Inside silos: silo host partition.
// - Inside hosted silo: host of the VM.
// - Inside VM: VM host.
// - Physical host: Not supported.
func HvsockGUIDParent() guid.GUID { // a42e7cda-d03f-480c-9cc2-a4de20abb878
return guid.GUID{
Data1: 0xa42e7cda,
Data2: 0xd03f,
Data3: 0x480c,
Data4: [8]uint8{0x9c, 0xc2, 0xa4, 0xde, 0x20, 0xab, 0xb8, 0x78},
}
}
// hvsockVsockServiceTemplate is the Service GUID used for the VSOCK protocol.
func hvsockVsockServiceTemplate() guid.GUID { // 00000000-facb-11e6-bd58-64006a7986d3
return guid.GUID{
Data2: 0xfacb,
Data3: 0x11e6,
Data4: [8]uint8{0xbd, 0x58, 0x64, 0x00, 0x6a, 0x79, 0x86, 0xd3},
}
}
// An HvsockAddr is an address for a AF_HYPERV socket.
type HvsockAddr struct {
@ -35,8 +108,10 @@ type rawHvsockAddr struct {
ServiceID guid.GUID
}
var _ socket.RawSockaddr = &rawHvsockAddr{}
// Network returns the address's network name, "hvsock".
func (addr *HvsockAddr) Network() string {
func (*HvsockAddr) Network() string {
return "hvsock"
}
@ -46,14 +121,14 @@ func (addr *HvsockAddr) String() string {
// VsockServiceID returns an hvsock service ID corresponding to the specified AF_VSOCK port.
func VsockServiceID(port uint32) guid.GUID {
g, _ := guid.FromString("00000000-facb-11e6-bd58-64006a7986d3")
g := hvsockVsockServiceTemplate() // make a copy
g.Data1 = port
return g
}
func (addr *HvsockAddr) raw() rawHvsockAddr {
return rawHvsockAddr{
Family: afHvSock,
Family: afHVSock,
VMID: addr.VMID,
ServiceID: addr.ServiceID,
}
@ -64,26 +139,54 @@ func (addr *HvsockAddr) fromRaw(raw *rawHvsockAddr) {
addr.ServiceID = raw.ServiceID
}
// Sockaddr returns a pointer to and the size of this struct.
//
// Implements the [socket.RawSockaddr] interface, and allows use in
// [socket.Bind] and [socket.ConnectEx].
func (r *rawHvsockAddr) Sockaddr() (unsafe.Pointer, int32, error) {
return unsafe.Pointer(r), int32(unsafe.Sizeof(rawHvsockAddr{})), nil
}
// Sockaddr interface allows use with `sockets.Bind()` and `.ConnectEx()`.
func (r *rawHvsockAddr) FromBytes(b []byte) error {
n := int(unsafe.Sizeof(rawHvsockAddr{}))
if len(b) < n {
return fmt.Errorf("got %d, want %d: %w", len(b), n, socket.ErrBufferSize)
}
copy(unsafe.Slice((*byte)(unsafe.Pointer(r)), n), b[:n])
if r.Family != afHVSock {
return fmt.Errorf("got %d, want %d: %w", r.Family, afHVSock, socket.ErrAddrFamily)
}
return nil
}
// HvsockListener is a socket listener for the AF_HYPERV address family.
type HvsockListener struct {
sock *win32File
addr HvsockAddr
}
var _ net.Listener = &HvsockListener{}
// HvsockConn is a connected socket of the AF_HYPERV address family.
type HvsockConn struct {
sock *win32File
local, remote HvsockAddr
}
func newHvSocket() (*win32File, error) {
fd, err := syscall.Socket(afHvSock, syscall.SOCK_STREAM, 1)
var _ net.Conn = &HvsockConn{}
func newHVSocket() (*win32File, error) {
fd, err := windows.Socket(afHVSock, windows.SOCK_STREAM, 1)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
f, err := makeWin32File(fd)
if err != nil {
syscall.Close(fd)
windows.Close(fd)
return nil, err
}
f.socket = true
@ -93,16 +196,24 @@ func newHvSocket() (*win32File, error) {
// ListenHvsock listens for connections on the specified hvsock address.
func ListenHvsock(addr *HvsockAddr) (_ *HvsockListener, err error) {
l := &HvsockListener{addr: *addr}
sock, err := newHvSocket()
var sock *win32File
sock, err = newHVSocket()
if err != nil {
return nil, l.opErr("listen", err)
}
defer func() {
if err != nil {
_ = sock.Close()
}
}()
sa := addr.raw()
err = bind(sock.handle, unsafe.Pointer(&sa), int32(unsafe.Sizeof(sa)))
err = socket.Bind(sock.handle, &sa)
if err != nil {
return nil, l.opErr("listen", os.NewSyscallError("socket", err))
}
err = syscall.Listen(sock.handle, 16)
err = windows.Listen(sock.handle, 16)
if err != nil {
return nil, l.opErr("listen", os.NewSyscallError("listen", err))
}
@ -120,7 +231,7 @@ func (l *HvsockListener) Addr() net.Addr {
// Accept waits for the next connection and returns it.
func (l *HvsockListener) Accept() (_ net.Conn, err error) {
sock, err := newHvSocket()
sock, err := newHVSocket()
if err != nil {
return nil, l.opErr("accept", err)
}
@ -129,27 +240,42 @@ func (l *HvsockListener) Accept() (_ net.Conn, err error) {
sock.Close()
}
}()
c, err := l.sock.prepareIo()
c, err := l.sock.prepareIO()
if err != nil {
return nil, l.opErr("accept", err)
}
defer l.sock.wg.Done()
// AcceptEx, per documentation, requires an extra 16 bytes per address.
//
// https://docs.microsoft.com/en-us/windows/win32/api/mswsock/nf-mswsock-acceptex
const addrlen = uint32(16 + unsafe.Sizeof(rawHvsockAddr{}))
var addrbuf [addrlen * 2]byte
var bytes uint32
err = syscall.AcceptEx(l.sock.handle, sock.handle, &addrbuf[0], 0, addrlen, addrlen, &bytes, &c.o)
_, err = l.sock.asyncIo(c, nil, bytes, err)
if err != nil {
err = windows.AcceptEx(l.sock.handle, sock.handle, &addrbuf[0], 0 /* rxdatalen */, addrlen, addrlen, &bytes, &c.o)
if _, err = l.sock.asyncIO(c, nil, bytes, err); err != nil {
return nil, l.opErr("accept", os.NewSyscallError("acceptex", err))
}
conn := &HvsockConn{
sock: sock,
}
// The local address returned in the AcceptEx buffer is the same as the Listener socket's
// address. However, the service GUID reported by GetSockName is different from the Listeners
// socket, and is sometimes the same as the local address of the socket that dialed the
// address, with the service GUID.Data1 incremented, but othertimes is different.
// todo: does the local address matter? is the listener's address or the actual address appropriate?
conn.local.fromRaw((*rawHvsockAddr)(unsafe.Pointer(&addrbuf[0])))
conn.remote.fromRaw((*rawHvsockAddr)(unsafe.Pointer(&addrbuf[addrlen])))
// initialize the accepted socket and update its properties with those of the listening socket
if err = windows.Setsockopt(sock.handle,
windows.SOL_SOCKET, windows.SO_UPDATE_ACCEPT_CONTEXT,
(*byte)(unsafe.Pointer(&l.sock.handle)), int32(unsafe.Sizeof(l.sock.handle))); err != nil {
return nil, conn.opErr("accept", os.NewSyscallError("setsockopt", err))
}
sock = nil
return conn, nil
}
@ -159,54 +285,183 @@ func (l *HvsockListener) Close() error {
return l.sock.Close()
}
/* Need to finish ConnectEx handling
func DialHvsock(ctx context.Context, addr *HvsockAddr) (*HvsockConn, error) {
sock, err := newHvSocket()
// HvsockDialer configures and dials a Hyper-V Socket (ie, [HvsockConn]).
type HvsockDialer struct {
// Deadline is the time the Dial operation must connect before erroring.
Deadline time.Time
// Retries is the number of additional connects to try if the connection times out, is refused,
// or the host is unreachable
Retries uint
// RetryWait is the time to wait after a connection error to retry
RetryWait time.Duration
rt *time.Timer // redial wait timer
}
// Dial the Hyper-V socket at addr.
//
// See [HvsockDialer.Dial] for more information.
func Dial(ctx context.Context, addr *HvsockAddr) (conn *HvsockConn, err error) {
return (&HvsockDialer{}).Dial(ctx, addr)
}
// Dial attempts to connect to the Hyper-V socket at addr, and returns a connection if successful.
// Will attempt (HvsockDialer).Retries if dialing fails, waiting (HvsockDialer).RetryWait between
// retries.
//
// Dialing can be cancelled either by providing (HvsockDialer).Deadline, or cancelling ctx.
func (d *HvsockDialer) Dial(ctx context.Context, addr *HvsockAddr) (conn *HvsockConn, err error) {
op := "dial"
// create the conn early to use opErr()
conn = &HvsockConn{
remote: *addr,
}
if !d.Deadline.IsZero() {
var cancel context.CancelFunc
ctx, cancel = context.WithDeadline(ctx, d.Deadline)
defer cancel()
}
// preemptive timeout/cancellation check
if err = ctx.Err(); err != nil {
return nil, conn.opErr(op, err)
}
sock, err := newHVSocket()
if err != nil {
return nil, err
return nil, conn.opErr(op, err)
}
defer func() {
if sock != nil {
sock.Close()
}
}()
c, err := sock.prepareIo()
sa := addr.raw()
err = socket.Bind(sock.handle, &sa)
if err != nil {
return nil, err
return nil, conn.opErr(op, os.NewSyscallError("bind", err))
}
c, err := sock.prepareIO()
if err != nil {
return nil, conn.opErr(op, err)
}
defer sock.wg.Done()
var bytes uint32
err = windows.ConnectEx(windows.Handle(sock.handle), sa, nil, 0, &bytes, &c.o)
_, err = sock.asyncIo(ctx, c, nil, bytes, err)
for i := uint(0); i <= d.Retries; i++ {
err = socket.ConnectEx(
sock.handle,
&sa,
nil, // sendBuf
0, // sendDataLen
&bytes,
(*windows.Overlapped)(unsafe.Pointer(&c.o)))
_, err = sock.asyncIO(c, nil, bytes, err)
if i < d.Retries && canRedial(err) {
if err = d.redialWait(ctx); err == nil {
continue
}
}
break
}
if err != nil {
return nil, err
return nil, conn.opErr(op, os.NewSyscallError("connectex", err))
}
conn := &HvsockConn{
sock: sock,
remote: *addr,
// update the connection properties, so shutdown can be used
if err = windows.Setsockopt(
sock.handle,
windows.SOL_SOCKET,
windows.SO_UPDATE_CONNECT_CONTEXT,
nil, // optvalue
0, // optlen
); err != nil {
return nil, conn.opErr(op, os.NewSyscallError("setsockopt", err))
}
// get the local name
var sal rawHvsockAddr
err = socket.GetSockName(sock.handle, &sal)
if err != nil {
return nil, conn.opErr(op, os.NewSyscallError("getsockname", err))
}
conn.local.fromRaw(&sal)
// one last check for timeout, since asyncIO doesn't check the context
if err = ctx.Err(); err != nil {
return nil, conn.opErr(op, err)
}
conn.sock = sock
sock = nil
return conn, nil
}
*/
// redialWait waits before attempting to redial, resetting the timer as appropriate.
func (d *HvsockDialer) redialWait(ctx context.Context) (err error) {
if d.RetryWait == 0 {
return nil
}
if d.rt == nil {
d.rt = time.NewTimer(d.RetryWait)
} else {
// should already be stopped and drained
d.rt.Reset(d.RetryWait)
}
select {
case <-ctx.Done():
case <-d.rt.C:
return nil
}
// stop and drain the timer
if !d.rt.Stop() {
<-d.rt.C
}
return ctx.Err()
}
// assumes error is a plain, unwrapped windows.Errno provided by direct syscall.
func canRedial(err error) bool {
//nolint:errorlint // guaranteed to be an Errno
switch err {
case windows.WSAECONNREFUSED, windows.WSAENETUNREACH, windows.WSAETIMEDOUT,
windows.ERROR_CONNECTION_REFUSED, windows.ERROR_CONNECTION_UNAVAIL:
return true
default:
return false
}
}
func (conn *HvsockConn) opErr(op string, err error) error {
// translate from "file closed" to "socket closed"
if errors.Is(err, ErrFileClosed) {
err = socket.ErrSocketClosed
}
return &net.OpError{Op: op, Net: "hvsock", Source: &conn.local, Addr: &conn.remote, Err: err}
}
func (conn *HvsockConn) Read(b []byte) (int, error) {
c, err := conn.sock.prepareIo()
c, err := conn.sock.prepareIO()
if err != nil {
return 0, conn.opErr("read", err)
}
defer conn.sock.wg.Done()
buf := syscall.WSABuf{Buf: &b[0], Len: uint32(len(b))}
buf := windows.WSABuf{Buf: &b[0], Len: uint32(len(b))}
var flags, bytes uint32
err = syscall.WSARecv(conn.sock.handle, &buf, 1, &bytes, &flags, &c.o, nil)
n, err := conn.sock.asyncIo(c, &conn.sock.readDeadline, bytes, err)
err = windows.WSARecv(conn.sock.handle, &buf, 1, &bytes, &flags, &c.o, nil)
n, err := conn.sock.asyncIO(c, &conn.sock.readDeadline, bytes, err)
if err != nil {
if _, ok := err.(syscall.Errno); ok {
err = os.NewSyscallError("wsarecv", err)
var eno windows.Errno
if errors.As(err, &eno) {
err = os.NewSyscallError("wsarecv", eno)
}
return 0, conn.opErr("read", err)
} else if n == 0 {
@ -229,18 +484,19 @@ func (conn *HvsockConn) Write(b []byte) (int, error) {
}
func (conn *HvsockConn) write(b []byte) (int, error) {
c, err := conn.sock.prepareIo()
c, err := conn.sock.prepareIO()
if err != nil {
return 0, conn.opErr("write", err)
}
defer conn.sock.wg.Done()
buf := syscall.WSABuf{Buf: &b[0], Len: uint32(len(b))}
buf := windows.WSABuf{Buf: &b[0], Len: uint32(len(b))}
var bytes uint32
err = syscall.WSASend(conn.sock.handle, &buf, 1, &bytes, 0, &c.o, nil)
n, err := conn.sock.asyncIo(c, &conn.sock.writeDeadline, bytes, err)
err = windows.WSASend(conn.sock.handle, &buf, 1, &bytes, 0, &c.o, nil)
n, err := conn.sock.asyncIO(c, &conn.sock.writeDeadline, bytes, err)
if err != nil {
if _, ok := err.(syscall.Errno); ok {
err = os.NewSyscallError("wsasend", err)
var eno windows.Errno
if errors.As(err, &eno) {
err = os.NewSyscallError("wsasend", eno)
}
return 0, conn.opErr("write", err)
}
@ -252,29 +508,43 @@ func (conn *HvsockConn) Close() error {
return conn.sock.Close()
}
func (conn *HvsockConn) IsClosed() bool {
return conn.sock.IsClosed()
}
// shutdown disables sending or receiving on a socket.
func (conn *HvsockConn) shutdown(how int) error {
err := syscall.Shutdown(conn.sock.handle, syscall.SHUT_RD)
if conn.IsClosed() {
return socket.ErrSocketClosed
}
err := windows.Shutdown(conn.sock.handle, how)
if err != nil {
// If the connection was closed, shutdowns fail with "not connected"
if errors.Is(err, windows.WSAENOTCONN) ||
errors.Is(err, windows.WSAESHUTDOWN) {
err = socket.ErrSocketClosed
}
return os.NewSyscallError("shutdown", err)
}
return nil
}
// CloseRead shuts down the read end of the socket.
// CloseRead shuts down the read end of the socket, preventing future read operations.
func (conn *HvsockConn) CloseRead() error {
err := conn.shutdown(syscall.SHUT_RD)
err := conn.shutdown(windows.SHUT_RD)
if err != nil {
return conn.opErr("close", err)
return conn.opErr("closeread", err)
}
return nil
}
// CloseWrite shuts down the write end of the socket, notifying the other endpoint that
// no more data will be written.
// CloseWrite shuts down the write end of the socket, preventing future write operations and
// notifying the other endpoint that no more data will be written.
func (conn *HvsockConn) CloseWrite() error {
err := conn.shutdown(syscall.SHUT_WR)
err := conn.shutdown(windows.SHUT_WR)
if err != nil {
return conn.opErr("close", err)
return conn.opErr("closewrite", err)
}
return nil
}
@ -291,8 +561,13 @@ func (conn *HvsockConn) RemoteAddr() net.Addr {
// SetDeadline implements the net.Conn SetDeadline method.
func (conn *HvsockConn) SetDeadline(t time.Time) error {
conn.SetReadDeadline(t)
conn.SetWriteDeadline(t)
// todo: implement `SetDeadline` for `win32File`
if err := conn.SetReadDeadline(t); err != nil {
return fmt.Errorf("set read deadline: %w", err)
}
if err := conn.SetWriteDeadline(t); err != nil {
return fmt.Errorf("set write deadline: %w", err)
}
return nil
}

View File

@ -0,0 +1,2 @@
// This package contains Win32 filesystem functionality.
package fs

262
vendor/github.com/Microsoft/go-winio/internal/fs/fs.go generated vendored Normal file
View File

@ -0,0 +1,262 @@
//go:build windows
package fs
import (
"golang.org/x/sys/windows"
"github.com/Microsoft/go-winio/internal/stringbuffer"
)
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go fs.go
// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew
//sys CreateFile(name string, access AccessMask, mode FileShareMode, sa *windows.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) [failretval==windows.InvalidHandle] = CreateFileW
const NullHandle windows.Handle = 0
// AccessMask defines standard, specific, and generic rights.
//
// Used with CreateFile and NtCreateFile (and co.).
//
// Bitmask:
// 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1
// 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0
// +---------------+---------------+-------------------------------+
// |G|G|G|G|Resvd|A| StandardRights| SpecificRights |
// |R|W|E|A| |S| | |
// +-+-------------+---------------+-------------------------------+
//
// GR Generic Read
// GW Generic Write
// GE Generic Exectue
// GA Generic All
// Resvd Reserved
// AS Access Security System
//
// https://learn.microsoft.com/en-us/windows/win32/secauthz/access-mask
//
// https://learn.microsoft.com/en-us/windows/win32/secauthz/generic-access-rights
//
// https://learn.microsoft.com/en-us/windows/win32/fileio/file-access-rights-constants
type AccessMask = windows.ACCESS_MASK
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
// Not actually any.
//
// For CreateFile: "query certain metadata such as file, directory, or device attributes without accessing that file or device"
// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilew#parameters
FILE_ANY_ACCESS AccessMask = 0
GENERIC_READ AccessMask = 0x8000_0000
GENERIC_WRITE AccessMask = 0x4000_0000
GENERIC_EXECUTE AccessMask = 0x2000_0000
GENERIC_ALL AccessMask = 0x1000_0000
ACCESS_SYSTEM_SECURITY AccessMask = 0x0100_0000
// Specific Object Access
// from ntioapi.h
FILE_READ_DATA AccessMask = (0x0001) // file & pipe
FILE_LIST_DIRECTORY AccessMask = (0x0001) // directory
FILE_WRITE_DATA AccessMask = (0x0002) // file & pipe
FILE_ADD_FILE AccessMask = (0x0002) // directory
FILE_APPEND_DATA AccessMask = (0x0004) // file
FILE_ADD_SUBDIRECTORY AccessMask = (0x0004) // directory
FILE_CREATE_PIPE_INSTANCE AccessMask = (0x0004) // named pipe
FILE_READ_EA AccessMask = (0x0008) // file & directory
FILE_READ_PROPERTIES AccessMask = FILE_READ_EA
FILE_WRITE_EA AccessMask = (0x0010) // file & directory
FILE_WRITE_PROPERTIES AccessMask = FILE_WRITE_EA
FILE_EXECUTE AccessMask = (0x0020) // file
FILE_TRAVERSE AccessMask = (0x0020) // directory
FILE_DELETE_CHILD AccessMask = (0x0040) // directory
FILE_READ_ATTRIBUTES AccessMask = (0x0080) // all
FILE_WRITE_ATTRIBUTES AccessMask = (0x0100) // all
FILE_ALL_ACCESS AccessMask = (STANDARD_RIGHTS_REQUIRED | SYNCHRONIZE | 0x1FF)
FILE_GENERIC_READ AccessMask = (STANDARD_RIGHTS_READ | FILE_READ_DATA | FILE_READ_ATTRIBUTES | FILE_READ_EA | SYNCHRONIZE)
FILE_GENERIC_WRITE AccessMask = (STANDARD_RIGHTS_WRITE | FILE_WRITE_DATA | FILE_WRITE_ATTRIBUTES | FILE_WRITE_EA | FILE_APPEND_DATA | SYNCHRONIZE)
FILE_GENERIC_EXECUTE AccessMask = (STANDARD_RIGHTS_EXECUTE | FILE_READ_ATTRIBUTES | FILE_EXECUTE | SYNCHRONIZE)
SPECIFIC_RIGHTS_ALL AccessMask = 0x0000FFFF
// Standard Access
// from ntseapi.h
DELETE AccessMask = 0x0001_0000
READ_CONTROL AccessMask = 0x0002_0000
WRITE_DAC AccessMask = 0x0004_0000
WRITE_OWNER AccessMask = 0x0008_0000
SYNCHRONIZE AccessMask = 0x0010_0000
STANDARD_RIGHTS_REQUIRED AccessMask = 0x000F_0000
STANDARD_RIGHTS_READ AccessMask = READ_CONTROL
STANDARD_RIGHTS_WRITE AccessMask = READ_CONTROL
STANDARD_RIGHTS_EXECUTE AccessMask = READ_CONTROL
STANDARD_RIGHTS_ALL AccessMask = 0x001F_0000
)
type FileShareMode uint32
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
FILE_SHARE_NONE FileShareMode = 0x00
FILE_SHARE_READ FileShareMode = 0x01
FILE_SHARE_WRITE FileShareMode = 0x02
FILE_SHARE_DELETE FileShareMode = 0x04
FILE_SHARE_VALID_FLAGS FileShareMode = 0x07
)
type FileCreationDisposition uint32
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
// from winbase.h
CREATE_NEW FileCreationDisposition = 0x01
CREATE_ALWAYS FileCreationDisposition = 0x02
OPEN_EXISTING FileCreationDisposition = 0x03
OPEN_ALWAYS FileCreationDisposition = 0x04
TRUNCATE_EXISTING FileCreationDisposition = 0x05
)
// Create disposition values for NtCreate*
type NTFileCreationDisposition uint32
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
// From ntioapi.h
FILE_SUPERSEDE NTFileCreationDisposition = 0x00
FILE_OPEN NTFileCreationDisposition = 0x01
FILE_CREATE NTFileCreationDisposition = 0x02
FILE_OPEN_IF NTFileCreationDisposition = 0x03
FILE_OVERWRITE NTFileCreationDisposition = 0x04
FILE_OVERWRITE_IF NTFileCreationDisposition = 0x05
FILE_MAXIMUM_DISPOSITION NTFileCreationDisposition = 0x05
)
// CreateFile and co. take flags or attributes together as one parameter.
// Define alias until we can use generics to allow both
//
// https://learn.microsoft.com/en-us/windows/win32/fileio/file-attribute-constants
type FileFlagOrAttribute uint32
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
// from winnt.h
FILE_FLAG_WRITE_THROUGH FileFlagOrAttribute = 0x8000_0000
FILE_FLAG_OVERLAPPED FileFlagOrAttribute = 0x4000_0000
FILE_FLAG_NO_BUFFERING FileFlagOrAttribute = 0x2000_0000
FILE_FLAG_RANDOM_ACCESS FileFlagOrAttribute = 0x1000_0000
FILE_FLAG_SEQUENTIAL_SCAN FileFlagOrAttribute = 0x0800_0000
FILE_FLAG_DELETE_ON_CLOSE FileFlagOrAttribute = 0x0400_0000
FILE_FLAG_BACKUP_SEMANTICS FileFlagOrAttribute = 0x0200_0000
FILE_FLAG_POSIX_SEMANTICS FileFlagOrAttribute = 0x0100_0000
FILE_FLAG_OPEN_REPARSE_POINT FileFlagOrAttribute = 0x0020_0000
FILE_FLAG_OPEN_NO_RECALL FileFlagOrAttribute = 0x0010_0000
FILE_FLAG_FIRST_PIPE_INSTANCE FileFlagOrAttribute = 0x0008_0000
)
// NtCreate* functions take a dedicated CreateOptions parameter.
//
// https://learn.microsoft.com/en-us/windows/win32/api/Winternl/nf-winternl-ntcreatefile
//
// https://learn.microsoft.com/en-us/windows/win32/devnotes/nt-create-named-pipe-file
type NTCreateOptions uint32
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
// From ntioapi.h
FILE_DIRECTORY_FILE NTCreateOptions = 0x0000_0001
FILE_WRITE_THROUGH NTCreateOptions = 0x0000_0002
FILE_SEQUENTIAL_ONLY NTCreateOptions = 0x0000_0004
FILE_NO_INTERMEDIATE_BUFFERING NTCreateOptions = 0x0000_0008
FILE_SYNCHRONOUS_IO_ALERT NTCreateOptions = 0x0000_0010
FILE_SYNCHRONOUS_IO_NONALERT NTCreateOptions = 0x0000_0020
FILE_NON_DIRECTORY_FILE NTCreateOptions = 0x0000_0040
FILE_CREATE_TREE_CONNECTION NTCreateOptions = 0x0000_0080
FILE_COMPLETE_IF_OPLOCKED NTCreateOptions = 0x0000_0100
FILE_NO_EA_KNOWLEDGE NTCreateOptions = 0x0000_0200
FILE_DISABLE_TUNNELING NTCreateOptions = 0x0000_0400
FILE_RANDOM_ACCESS NTCreateOptions = 0x0000_0800
FILE_DELETE_ON_CLOSE NTCreateOptions = 0x0000_1000
FILE_OPEN_BY_FILE_ID NTCreateOptions = 0x0000_2000
FILE_OPEN_FOR_BACKUP_INTENT NTCreateOptions = 0x0000_4000
FILE_NO_COMPRESSION NTCreateOptions = 0x0000_8000
)
type FileSQSFlag = FileFlagOrAttribute
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
// from winbase.h
SECURITY_ANONYMOUS FileSQSFlag = FileSQSFlag(SecurityAnonymous << 16)
SECURITY_IDENTIFICATION FileSQSFlag = FileSQSFlag(SecurityIdentification << 16)
SECURITY_IMPERSONATION FileSQSFlag = FileSQSFlag(SecurityImpersonation << 16)
SECURITY_DELEGATION FileSQSFlag = FileSQSFlag(SecurityDelegation << 16)
SECURITY_SQOS_PRESENT FileSQSFlag = 0x0010_0000
SECURITY_VALID_SQOS_FLAGS FileSQSFlag = 0x001F_0000
)
// GetFinalPathNameByHandle flags
//
// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfinalpathnamebyhandlew#parameters
type GetFinalPathFlag uint32
//nolint:revive // SNAKE_CASE is not idiomatic in Go, but aligned with Win32 API.
const (
GetFinalPathDefaultFlag GetFinalPathFlag = 0x0
FILE_NAME_NORMALIZED GetFinalPathFlag = 0x0
FILE_NAME_OPENED GetFinalPathFlag = 0x8
VOLUME_NAME_DOS GetFinalPathFlag = 0x0
VOLUME_NAME_GUID GetFinalPathFlag = 0x1
VOLUME_NAME_NT GetFinalPathFlag = 0x2
VOLUME_NAME_NONE GetFinalPathFlag = 0x4
)
// getFinalPathNameByHandle facilitates calling the Windows API GetFinalPathNameByHandle
// with the given handle and flags. It transparently takes care of creating a buffer of the
// correct size for the call.
//
// https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfinalpathnamebyhandlew
func GetFinalPathNameByHandle(h windows.Handle, flags GetFinalPathFlag) (string, error) {
b := stringbuffer.NewWString()
//TODO: can loop infinitely if Win32 keeps returning the same (or a larger) n?
for {
n, err := windows.GetFinalPathNameByHandle(h, b.Pointer(), b.Cap(), uint32(flags))
if err != nil {
return "", err
}
// If the buffer wasn't large enough, n will be the total size needed (including null terminator).
// Resize and try again.
if n > b.Cap() {
b.ResizeTo(n)
continue
}
// If the buffer is large enough, n will be the size not including the null terminator.
// Convert to a Go string and return.
return b.String(), nil
}
}

View File

@ -0,0 +1,12 @@
package fs
// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ne-winnt-security_impersonation_level
type SecurityImpersonationLevel int32 // C default enums underlying type is `int`, which is Go `int32`
// Impersonation levels
const (
SecurityAnonymous SecurityImpersonationLevel = 0
SecurityIdentification SecurityImpersonationLevel = 1
SecurityImpersonation SecurityImpersonationLevel = 2
SecurityDelegation SecurityImpersonationLevel = 3
)

View File

@ -0,0 +1,61 @@
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package fs
import (
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
var _ unsafe.Pointer
// Do the interface allocations only once for common
// Errno values.
const (
errnoERROR_IO_PENDING = 997
)
var (
errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
errERROR_EINVAL error = syscall.EINVAL
)
// errnoErr returns common boxed Errno values, to prevent
// allocations at runtime.
func errnoErr(e syscall.Errno) error {
switch e {
case 0:
return errERROR_EINVAL
case errnoERROR_IO_PENDING:
return errERROR_IO_PENDING
}
return e
}
var (
modkernel32 = windows.NewLazySystemDLL("kernel32.dll")
procCreateFileW = modkernel32.NewProc("CreateFileW")
)
func CreateFile(name string, access AccessMask, mode FileShareMode, sa *windows.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) {
var _p0 *uint16
_p0, err = syscall.UTF16PtrFromString(name)
if err != nil {
return
}
return _CreateFile(_p0, access, mode, sa, createmode, attrs, templatefile)
}
func _CreateFile(name *uint16, access AccessMask, mode FileShareMode, sa *windows.SecurityAttributes, createmode FileCreationDisposition, attrs FileFlagOrAttribute, templatefile windows.Handle) (handle windows.Handle, err error) {
r0, _, e1 := syscall.SyscallN(procCreateFileW.Addr(), uintptr(unsafe.Pointer(name)), uintptr(access), uintptr(mode), uintptr(unsafe.Pointer(sa)), uintptr(createmode), uintptr(attrs), uintptr(templatefile))
handle = windows.Handle(r0)
if handle == windows.InvalidHandle {
err = errnoErr(e1)
}
return
}

View File

@ -0,0 +1,20 @@
package socket
import (
"unsafe"
)
// RawSockaddr allows structs to be used with [Bind] and [ConnectEx]. The
// struct must meet the Win32 sockaddr requirements specified here:
// https://docs.microsoft.com/en-us/windows/win32/winsock/sockaddr-2
//
// Specifically, the struct size must be least larger than an int16 (unsigned short)
// for the address family.
type RawSockaddr interface {
// Sockaddr returns a pointer to the RawSockaddr and its struct size, allowing
// for the RawSockaddr's data to be overwritten by syscalls (if necessary).
//
// It is the callers responsibility to validate that the values are valid; invalid
// pointers or size can cause a panic.
Sockaddr() (unsafe.Pointer, int32, error)
}

View File

@ -0,0 +1,177 @@
//go:build windows
package socket
import (
"errors"
"fmt"
"net"
"sync"
"syscall"
"unsafe"
"github.com/Microsoft/go-winio/pkg/guid"
"golang.org/x/sys/windows"
)
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go socket.go
//sys getsockname(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) [failretval==socketError] = ws2_32.getsockname
//sys getpeername(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) [failretval==socketError] = ws2_32.getpeername
//sys bind(s windows.Handle, name unsafe.Pointer, namelen int32) (err error) [failretval==socketError] = ws2_32.bind
const socketError = uintptr(^uint32(0))
var (
// todo(helsaawy): create custom error types to store the desired vs actual size and addr family?
ErrBufferSize = errors.New("buffer size")
ErrAddrFamily = errors.New("address family")
ErrInvalidPointer = errors.New("invalid pointer")
ErrSocketClosed = fmt.Errorf("socket closed: %w", net.ErrClosed)
)
// todo(helsaawy): replace these with generics, ie: GetSockName[S RawSockaddr](s windows.Handle) (S, error)
// GetSockName writes the local address of socket s to the [RawSockaddr] rsa.
// If rsa is not large enough, the [windows.WSAEFAULT] is returned.
func GetSockName(s windows.Handle, rsa RawSockaddr) error {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
// although getsockname returns WSAEFAULT if the buffer is too small, it does not set
// &l to the correct size, so--apart from doubling the buffer repeatedly--there is no remedy
return getsockname(s, ptr, &l)
}
// GetPeerName returns the remote address the socket is connected to.
//
// See [GetSockName] for more information.
func GetPeerName(s windows.Handle, rsa RawSockaddr) error {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
return getpeername(s, ptr, &l)
}
func Bind(s windows.Handle, rsa RawSockaddr) (err error) {
ptr, l, err := rsa.Sockaddr()
if err != nil {
return fmt.Errorf("could not retrieve socket pointer and size: %w", err)
}
return bind(s, ptr, l)
}
// "golang.org/x/sys/windows".ConnectEx and .Bind only accept internal implementations of the
// their sockaddr interface, so they cannot be used with HvsockAddr
// Replicate functionality here from
// https://cs.opensource.google/go/x/sys/+/master:windows/syscall_windows.go
// The function pointers to `AcceptEx`, `ConnectEx` and `GetAcceptExSockaddrs` must be loaded at
// runtime via a WSAIoctl call:
// https://docs.microsoft.com/en-us/windows/win32/api/Mswsock/nc-mswsock-lpfn_connectex#remarks
type runtimeFunc struct {
id guid.GUID
once sync.Once
addr uintptr
err error
}
func (f *runtimeFunc) Load() error {
f.once.Do(func() {
var s windows.Handle
s, f.err = windows.Socket(windows.AF_INET, windows.SOCK_STREAM, windows.IPPROTO_TCP)
if f.err != nil {
return
}
defer windows.CloseHandle(s) //nolint:errcheck
var n uint32
f.err = windows.WSAIoctl(s,
windows.SIO_GET_EXTENSION_FUNCTION_POINTER,
(*byte)(unsafe.Pointer(&f.id)),
uint32(unsafe.Sizeof(f.id)),
(*byte)(unsafe.Pointer(&f.addr)),
uint32(unsafe.Sizeof(f.addr)),
&n,
nil, // overlapped
0, // completionRoutine
)
})
return f.err
}
var (
// todo: add `AcceptEx` and `GetAcceptExSockaddrs`
WSAID_CONNECTEX = guid.GUID{ //revive:disable-line:var-naming ALL_CAPS
Data1: 0x25a207b9,
Data2: 0xddf3,
Data3: 0x4660,
Data4: [8]byte{0x8e, 0xe9, 0x76, 0xe5, 0x8c, 0x74, 0x06, 0x3e},
}
connectExFunc = runtimeFunc{id: WSAID_CONNECTEX}
)
func ConnectEx(
fd windows.Handle,
rsa RawSockaddr,
sendBuf *byte,
sendDataLen uint32,
bytesSent *uint32,
overlapped *windows.Overlapped,
) error {
if err := connectExFunc.Load(); err != nil {
return fmt.Errorf("failed to load ConnectEx function pointer: %w", err)
}
ptr, n, err := rsa.Sockaddr()
if err != nil {
return err
}
return connectEx(fd, ptr, n, sendBuf, sendDataLen, bytesSent, overlapped)
}
// BOOL LpfnConnectex(
// [in] SOCKET s,
// [in] const sockaddr *name,
// [in] int namelen,
// [in, optional] PVOID lpSendBuffer,
// [in] DWORD dwSendDataLength,
// [out] LPDWORD lpdwBytesSent,
// [in] LPOVERLAPPED lpOverlapped
// )
func connectEx(
s windows.Handle,
name unsafe.Pointer,
namelen int32,
sendBuf *byte,
sendDataLen uint32,
bytesSent *uint32,
overlapped *windows.Overlapped,
) (err error) {
r1, _, e1 := syscall.SyscallN(connectExFunc.addr,
uintptr(s),
uintptr(name),
uintptr(namelen),
uintptr(unsafe.Pointer(sendBuf)),
uintptr(sendDataLen),
uintptr(unsafe.Pointer(bytesSent)),
uintptr(unsafe.Pointer(overlapped)),
)
if r1 == 0 {
if e1 != 0 {
err = error(e1)
} else {
err = syscall.EINVAL
}
}
return err
}

View File

@ -0,0 +1,69 @@
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package socket
import (
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
var _ unsafe.Pointer
// Do the interface allocations only once for common
// Errno values.
const (
errnoERROR_IO_PENDING = 997
)
var (
errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
errERROR_EINVAL error = syscall.EINVAL
)
// errnoErr returns common boxed Errno values, to prevent
// allocations at runtime.
func errnoErr(e syscall.Errno) error {
switch e {
case 0:
return errERROR_EINVAL
case errnoERROR_IO_PENDING:
return errERROR_IO_PENDING
}
return e
}
var (
modws2_32 = windows.NewLazySystemDLL("ws2_32.dll")
procbind = modws2_32.NewProc("bind")
procgetpeername = modws2_32.NewProc("getpeername")
procgetsockname = modws2_32.NewProc("getsockname")
)
func bind(s windows.Handle, name unsafe.Pointer, namelen int32) (err error) {
r1, _, e1 := syscall.SyscallN(procbind.Addr(), uintptr(s), uintptr(name), uintptr(namelen))
if r1 == socketError {
err = errnoErr(e1)
}
return
}
func getpeername(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) {
r1, _, e1 := syscall.SyscallN(procgetpeername.Addr(), uintptr(s), uintptr(name), uintptr(unsafe.Pointer(namelen)))
if r1 == socketError {
err = errnoErr(e1)
}
return
}
func getsockname(s windows.Handle, name unsafe.Pointer, namelen *int32) (err error) {
r1, _, e1 := syscall.SyscallN(procgetsockname.Addr(), uintptr(s), uintptr(name), uintptr(unsafe.Pointer(namelen)))
if r1 == socketError {
err = errnoErr(e1)
}
return
}

View File

@ -0,0 +1,132 @@
package stringbuffer
import (
"sync"
"unicode/utf16"
)
// TODO: worth exporting and using in mkwinsyscall?
// Uint16BufferSize is the buffer size in the pool, chosen somewhat arbitrarily to accommodate
// large path strings:
// MAX_PATH (260) + size of volume GUID prefix (49) + null terminator = 310.
const MinWStringCap = 310
// use *[]uint16 since []uint16 creates an extra allocation where the slice header
// is copied to heap and then referenced via pointer in the interface header that sync.Pool
// stores.
var pathPool = sync.Pool{ // if go1.18+ adds Pool[T], use that to store []uint16 directly
New: func() interface{} {
b := make([]uint16, MinWStringCap)
return &b
},
}
func newBuffer() []uint16 { return *(pathPool.Get().(*[]uint16)) }
// freeBuffer copies the slice header data, and puts a pointer to that in the pool.
// This avoids taking a pointer to the slice header in WString, which can be set to nil.
func freeBuffer(b []uint16) { pathPool.Put(&b) }
// WString is a wide string buffer ([]uint16) meant for storing UTF-16 encoded strings
// for interacting with Win32 APIs.
// Sizes are specified as uint32 and not int.
//
// It is not thread safe.
type WString struct {
// type-def allows casting to []uint16 directly, use struct to prevent that and allow adding fields in the future.
// raw buffer
b []uint16
}
// NewWString returns a [WString] allocated from a shared pool with an
// initial capacity of at least [MinWStringCap].
// Since the buffer may have been previously used, its contents are not guaranteed to be empty.
//
// The buffer should be freed via [WString.Free]
func NewWString() *WString {
return &WString{
b: newBuffer(),
}
}
func (b *WString) Free() {
if b.empty() {
return
}
freeBuffer(b.b)
b.b = nil
}
// ResizeTo grows the buffer to at least c and returns the new capacity, freeing the
// previous buffer back into pool.
func (b *WString) ResizeTo(c uint32) uint32 {
// already sufficient (or n is 0)
if c <= b.Cap() {
return b.Cap()
}
if c <= MinWStringCap {
c = MinWStringCap
}
// allocate at-least double buffer size, as is done in [bytes.Buffer] and other places
if c <= 2*b.Cap() {
c = 2 * b.Cap()
}
b2 := make([]uint16, c)
if !b.empty() {
copy(b2, b.b)
freeBuffer(b.b)
}
b.b = b2
return c
}
// Buffer returns the underlying []uint16 buffer.
func (b *WString) Buffer() []uint16 {
if b.empty() {
return nil
}
return b.b
}
// Pointer returns a pointer to the first uint16 in the buffer.
// If the [WString.Free] has already been called, the pointer will be nil.
func (b *WString) Pointer() *uint16 {
if b.empty() {
return nil
}
return &b.b[0]
}
// String returns the returns the UTF-8 encoding of the UTF-16 string in the buffer.
//
// It assumes that the data is null-terminated.
func (b *WString) String() string {
// Using [windows.UTF16ToString] would require importing "golang.org/x/sys/windows"
// and would make this code Windows-only, which makes no sense.
// So copy UTF16ToString code into here.
// If other windows-specific code is added, switch to [windows.UTF16ToString]
s := b.b
for i, v := range s {
if v == 0 {
s = s[:i]
break
}
}
return string(utf16.Decode(s))
}
// Cap returns the underlying buffer capacity.
func (b *WString) Cap() uint32 {
if b.empty() {
return 0
}
return b.cap()
}
func (b *WString) cap() uint32 { return uint32(cap(b.b)) }
func (b *WString) empty() bool { return b == nil || b.cap() == 0 }

View File

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@ -10,26 +11,52 @@ import (
"net"
"os"
"runtime"
"syscall"
"time"
"unsafe"
"golang.org/x/sys/windows"
"github.com/Microsoft/go-winio/internal/fs"
)
//sys connectNamedPipe(pipe syscall.Handle, o *syscall.Overlapped) (err error) = ConnectNamedPipe
//sys createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *syscall.SecurityAttributes) (handle syscall.Handle, err error) [failretval==syscall.InvalidHandle] = CreateNamedPipeW
//sys createFile(name string, access uint32, mode uint32, sa *syscall.SecurityAttributes, createmode uint32, attrs uint32, templatefile syscall.Handle) (handle syscall.Handle, err error) [failretval==syscall.InvalidHandle] = CreateFileW
//sys getNamedPipeInfo(pipe syscall.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) = GetNamedPipeInfo
//sys getNamedPipeHandleState(pipe syscall.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW
//sys localAlloc(uFlags uint32, length uint32) (ptr uintptr) = LocalAlloc
//sys ntCreateNamedPipeFile(pipe *syscall.Handle, access uint32, oa *objectAttributes, iosb *ioStatusBlock, share uint32, disposition uint32, options uint32, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntstatus) = ntdll.NtCreateNamedPipeFile
//sys rtlNtStatusToDosError(status ntstatus) (winerr error) = ntdll.RtlNtStatusToDosErrorNoTeb
//sys rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntstatus) = ntdll.RtlDosPathNameToNtPathName_U
//sys rtlDefaultNpAcl(dacl *uintptr) (status ntstatus) = ntdll.RtlDefaultNpAcl
//sys connectNamedPipe(pipe windows.Handle, o *windows.Overlapped) (err error) = ConnectNamedPipe
//sys createNamedPipe(name string, flags uint32, pipeMode uint32, maxInstances uint32, outSize uint32, inSize uint32, defaultTimeout uint32, sa *windows.SecurityAttributes) (handle windows.Handle, err error) [failretval==windows.InvalidHandle] = CreateNamedPipeW
//sys disconnectNamedPipe(pipe windows.Handle) (err error) = DisconnectNamedPipe
//sys getNamedPipeInfo(pipe windows.Handle, flags *uint32, outSize *uint32, inSize *uint32, maxInstances *uint32) (err error) = GetNamedPipeInfo
//sys getNamedPipeHandleState(pipe windows.Handle, state *uint32, curInstances *uint32, maxCollectionCount *uint32, collectDataTimeout *uint32, userName *uint16, maxUserNameSize uint32) (err error) = GetNamedPipeHandleStateW
//sys ntCreateNamedPipeFile(pipe *windows.Handle, access ntAccessMask, oa *objectAttributes, iosb *ioStatusBlock, share ntFileShareMode, disposition ntFileCreationDisposition, options ntFileOptions, typ uint32, readMode uint32, completionMode uint32, maxInstances uint32, inboundQuota uint32, outputQuota uint32, timeout *int64) (status ntStatus) = ntdll.NtCreateNamedPipeFile
//sys rtlNtStatusToDosError(status ntStatus) (winerr error) = ntdll.RtlNtStatusToDosErrorNoTeb
//sys rtlDosPathNameToNtPathName(name *uint16, ntName *unicodeString, filePart uintptr, reserved uintptr) (status ntStatus) = ntdll.RtlDosPathNameToNtPathName_U
//sys rtlDefaultNpAcl(dacl *uintptr) (status ntStatus) = ntdll.RtlDefaultNpAcl
type PipeConn interface {
net.Conn
Disconnect() error
Flush() error
}
// type aliases for mkwinsyscall code
type (
ntAccessMask = fs.AccessMask
ntFileShareMode = fs.FileShareMode
ntFileCreationDisposition = fs.NTFileCreationDisposition
ntFileOptions = fs.NTCreateOptions
)
type ioStatusBlock struct {
Status, Information uintptr
}
// typedef struct _OBJECT_ATTRIBUTES {
// ULONG Length;
// HANDLE RootDirectory;
// PUNICODE_STRING ObjectName;
// ULONG Attributes;
// PVOID SecurityDescriptor;
// PVOID SecurityQualityOfService;
// } OBJECT_ATTRIBUTES;
//
// https://learn.microsoft.com/en-us/windows/win32/api/ntdef/ns-ntdef-_object_attributes
type objectAttributes struct {
Length uintptr
RootDirectory uintptr
@ -45,51 +72,39 @@ type unicodeString struct {
Buffer uintptr
}
// typedef struct _SECURITY_DESCRIPTOR {
// BYTE Revision;
// BYTE Sbz1;
// SECURITY_DESCRIPTOR_CONTROL Control;
// PSID Owner;
// PSID Group;
// PACL Sacl;
// PACL Dacl;
// } SECURITY_DESCRIPTOR, *PISECURITY_DESCRIPTOR;
//
// https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-security_descriptor
type securityDescriptor struct {
Revision byte
Sbz1 byte
Control uint16
Owner uintptr
Group uintptr
Sacl uintptr
Dacl uintptr
Sacl uintptr //revive:disable-line:var-naming SACL, not Sacl
Dacl uintptr //revive:disable-line:var-naming DACL, not Dacl
}
type ntstatus int32
type ntStatus int32
func (status ntstatus) Err() error {
func (status ntStatus) Err() error {
if status >= 0 {
return nil
}
return rtlNtStatusToDosError(status)
}
const (
cERROR_PIPE_BUSY = syscall.Errno(231)
cERROR_NO_DATA = syscall.Errno(232)
cERROR_PIPE_CONNECTED = syscall.Errno(535)
cERROR_SEM_TIMEOUT = syscall.Errno(121)
cSECURITY_SQOS_PRESENT = 0x100000
cSECURITY_ANONYMOUS = 0
cPIPE_TYPE_MESSAGE = 4
cPIPE_READMODE_MESSAGE = 2
cFILE_OPEN = 1
cFILE_CREATE = 2
cFILE_PIPE_MESSAGE_TYPE = 1
cFILE_PIPE_REJECT_REMOTE_CLIENTS = 2
cSE_DACL_PRESENT = 4
)
var (
// ErrPipeListenerClosed is returned for pipe operations on listeners that have been closed.
// This error should match net.errClosing since docker takes a dependency on its text.
ErrPipeListenerClosed = errors.New("use of closed network connection")
ErrPipeListenerClosed = net.ErrClosed
errPipeWriteClosed = errors.New("pipe has been closed for write")
)
@ -99,6 +114,8 @@ type win32Pipe struct {
path string
}
var _ PipeConn = (*win32Pipe)(nil)
type win32MessageBytePipe struct {
win32Pipe
writeClosed bool
@ -116,9 +133,14 @@ func (f *win32Pipe) RemoteAddr() net.Addr {
}
func (f *win32Pipe) SetDeadline(t time.Time) error {
f.SetReadDeadline(t)
f.SetWriteDeadline(t)
return nil
if err := f.SetReadDeadline(t); err != nil {
return err
}
return f.SetWriteDeadline(t)
}
func (f *win32Pipe) Disconnect() error {
return disconnectNamedPipe(f.win32File.handle)
}
// CloseWrite closes the write side of a message pipe in byte mode.
@ -157,14 +179,14 @@ func (f *win32MessageBytePipe) Read(b []byte) (int, error) {
return 0, io.EOF
}
n, err := f.win32File.Read(b)
if err == io.EOF {
if err == io.EOF { //nolint:errorlint
// If this was the result of a zero-byte read, then
// it is possible that the read was due to a zero-size
// message. Since we are simulating CloseWrite with a
// zero-byte message, ensure that all future Read() calls
// also return EOF.
f.readEOF = true
} else if err == syscall.ERROR_MORE_DATA {
} else if err == windows.ERROR_MORE_DATA { //nolint:errorlint // err is Errno
// ERROR_MORE_DATA indicates that the pipe's read mode is message mode
// and the message still has more bytes. Treat this as a success, since
// this package presents all named pipes as byte streams.
@ -173,7 +195,7 @@ func (f *win32MessageBytePipe) Read(b []byte) (int, error) {
return n, err
}
func (s pipeAddress) Network() string {
func (pipeAddress) Network() string {
return "pipe"
}
@ -182,18 +204,24 @@ func (s pipeAddress) String() string {
}
// tryDialPipe attempts to dial the pipe at `path` until `ctx` cancellation or timeout.
func tryDialPipe(ctx context.Context, path *string, access uint32) (syscall.Handle, error) {
func tryDialPipe(ctx context.Context, path *string, access fs.AccessMask, impLevel PipeImpLevel) (windows.Handle, error) {
for {
select {
case <-ctx.Done():
return syscall.Handle(0), ctx.Err()
return windows.Handle(0), ctx.Err()
default:
h, err := createFile(*path, access, 0, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_OVERLAPPED|cSECURITY_SQOS_PRESENT|cSECURITY_ANONYMOUS, 0)
h, err := fs.CreateFile(*path,
access,
0, // mode
nil, // security attributes
fs.OPEN_EXISTING,
fs.FILE_FLAG_OVERLAPPED|fs.SECURITY_SQOS_PRESENT|fs.FileSQSFlag(impLevel),
0, // template file handle
)
if err == nil {
return h, nil
}
if err != cERROR_PIPE_BUSY {
if err != windows.ERROR_PIPE_BUSY { //nolint:errorlint // err is Errno
return h, &os.PathError{Err: err, Op: "open", Path: *path}
}
// Wait 10 msec and try again. This is a rather simplistic
@ -213,9 +241,10 @@ func DialPipe(path string, timeout *time.Duration) (net.Conn, error) {
} else {
absTimeout = time.Now().Add(2 * time.Second)
}
ctx, _ := context.WithDeadline(context.Background(), absTimeout)
ctx, cancel := context.WithDeadline(context.Background(), absTimeout)
defer cancel()
conn, err := DialPipeContext(ctx, path)
if err == context.DeadlineExceeded {
if errors.Is(err, context.DeadlineExceeded) {
return nil, ErrTimeout
}
return conn, err
@ -224,15 +253,33 @@ func DialPipe(path string, timeout *time.Duration) (net.Conn, error) {
// DialPipeContext attempts to connect to a named pipe by `path` until `ctx`
// cancellation or timeout.
func DialPipeContext(ctx context.Context, path string) (net.Conn, error) {
return DialPipeAccess(ctx, path, syscall.GENERIC_READ|syscall.GENERIC_WRITE)
return DialPipeAccess(ctx, path, uint32(fs.GENERIC_READ|fs.GENERIC_WRITE))
}
// PipeImpLevel is an enumeration of impersonation levels that may be set
// when calling DialPipeAccessImpersonation.
type PipeImpLevel uint32
const (
PipeImpLevelAnonymous = PipeImpLevel(fs.SECURITY_ANONYMOUS)
PipeImpLevelIdentification = PipeImpLevel(fs.SECURITY_IDENTIFICATION)
PipeImpLevelImpersonation = PipeImpLevel(fs.SECURITY_IMPERSONATION)
PipeImpLevelDelegation = PipeImpLevel(fs.SECURITY_DELEGATION)
)
// DialPipeAccess attempts to connect to a named pipe by `path` with `access` until `ctx`
// cancellation or timeout.
func DialPipeAccess(ctx context.Context, path string, access uint32) (net.Conn, error) {
return DialPipeAccessImpLevel(ctx, path, access, PipeImpLevelAnonymous)
}
// DialPipeAccessImpLevel attempts to connect to a named pipe by `path` with
// `access` at `impLevel` until `ctx` cancellation or timeout. The other
// DialPipe* implementations use PipeImpLevelAnonymous.
func DialPipeAccessImpLevel(ctx context.Context, path string, access uint32, impLevel PipeImpLevel) (net.Conn, error) {
var err error
var h syscall.Handle
h, err = tryDialPipe(ctx, &path, access)
var h windows.Handle
h, err = tryDialPipe(ctx, &path, fs.AccessMask(access), impLevel)
if err != nil {
return nil, err
}
@ -245,13 +292,13 @@ func DialPipeAccess(ctx context.Context, path string, access uint32) (net.Conn,
f, err := makeWin32File(h)
if err != nil {
syscall.Close(h)
windows.Close(h)
return nil, err
}
// If the pipe is in message mode, return a message byte pipe, which
// supports CloseWrite().
if flags&cPIPE_TYPE_MESSAGE != 0 {
if flags&windows.PIPE_TYPE_MESSAGE != 0 {
return &win32MessageBytePipe{
win32Pipe: win32Pipe{win32File: f, path: path},
}, nil
@ -265,7 +312,7 @@ type acceptResponse struct {
}
type win32PipeListener struct {
firstHandle syscall.Handle
firstHandle windows.Handle
path string
config PipeConfig
acceptCh chan (chan acceptResponse)
@ -273,8 +320,8 @@ type win32PipeListener struct {
doneCh chan int
}
func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (syscall.Handle, error) {
path16, err := syscall.UTF16FromString(path)
func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (windows.Handle, error) {
path16, err := windows.UTF16FromString(path)
if err != nil {
return 0, &os.PathError{Op: "open", Path: path, Err: err}
}
@ -283,59 +330,81 @@ func makeServerPipeHandle(path string, sd []byte, c *PipeConfig, first bool) (sy
oa.Length = unsafe.Sizeof(oa)
var ntPath unicodeString
if err := rtlDosPathNameToNtPathName(&path16[0], &ntPath, 0, 0).Err(); err != nil {
if err := rtlDosPathNameToNtPathName(&path16[0],
&ntPath,
0,
0,
).Err(); err != nil {
return 0, &os.PathError{Op: "open", Path: path, Err: err}
}
defer localFree(ntPath.Buffer)
defer windows.LocalFree(windows.Handle(ntPath.Buffer)) //nolint:errcheck
oa.ObjectName = &ntPath
oa.Attributes = windows.OBJ_CASE_INSENSITIVE
// The security descriptor is only needed for the first pipe.
if first {
if sd != nil {
len := uint32(len(sd))
sdb := localAlloc(0, len)
defer localFree(sdb)
//todo: does `sdb` need to be allocated on the heap, or can go allocate it?
l := uint32(len(sd))
sdb, err := windows.LocalAlloc(0, l)
if err != nil {
return 0, fmt.Errorf("LocalAlloc for security descriptor with of length %d: %w", l, err)
}
defer windows.LocalFree(windows.Handle(sdb)) //nolint:errcheck
copy((*[0xffff]byte)(unsafe.Pointer(sdb))[:], sd)
oa.SecurityDescriptor = (*securityDescriptor)(unsafe.Pointer(sdb))
} else {
// Construct the default named pipe security descriptor.
var dacl uintptr
if err := rtlDefaultNpAcl(&dacl).Err(); err != nil {
return 0, fmt.Errorf("getting default named pipe ACL: %s", err)
return 0, fmt.Errorf("getting default named pipe ACL: %w", err)
}
defer localFree(dacl)
defer windows.LocalFree(windows.Handle(dacl)) //nolint:errcheck
sdb := &securityDescriptor{
Revision: 1,
Control: cSE_DACL_PRESENT,
Control: windows.SE_DACL_PRESENT,
Dacl: dacl,
}
oa.SecurityDescriptor = sdb
}
}
typ := uint32(cFILE_PIPE_REJECT_REMOTE_CLIENTS)
typ := uint32(windows.FILE_PIPE_REJECT_REMOTE_CLIENTS)
if c.MessageMode {
typ |= cFILE_PIPE_MESSAGE_TYPE
typ |= windows.FILE_PIPE_MESSAGE_TYPE
}
disposition := uint32(cFILE_OPEN)
access := uint32(syscall.GENERIC_READ | syscall.GENERIC_WRITE | syscall.SYNCHRONIZE)
disposition := fs.FILE_OPEN
access := fs.GENERIC_READ | fs.GENERIC_WRITE | fs.SYNCHRONIZE
if first {
disposition = cFILE_CREATE
disposition = fs.FILE_CREATE
// By not asking for read or write access, the named pipe file system
// will put this pipe into an initially disconnected state, blocking
// client connections until the next call with first == false.
access = syscall.SYNCHRONIZE
access = fs.SYNCHRONIZE
}
timeout := int64(-50 * 10000) // 50ms
var (
h syscall.Handle
h windows.Handle
iosb ioStatusBlock
)
err = ntCreateNamedPipeFile(&h, access, &oa, &iosb, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, disposition, 0, typ, 0, 0, 0xffffffff, uint32(c.InputBufferSize), uint32(c.OutputBufferSize), &timeout).Err()
err = ntCreateNamedPipeFile(&h,
access,
&oa,
&iosb,
fs.FILE_SHARE_READ|fs.FILE_SHARE_WRITE,
disposition,
0,
typ,
0,
0,
0xffffffff,
uint32(c.InputBufferSize),
uint32(c.OutputBufferSize),
&timeout).Err()
if err != nil {
return 0, &os.PathError{Op: "open", Path: path, Err: err}
}
@ -351,7 +420,7 @@ func (l *win32PipeListener) makeServerPipe() (*win32File, error) {
}
f, err := makeWin32File(h)
if err != nil {
syscall.Close(h)
windows.Close(h)
return nil, err
}
return f, nil
@ -380,7 +449,7 @@ func (l *win32PipeListener) makeConnectedServerPipe() (*win32File, error) {
p.Close()
p = nil
err = <-ch
if err == nil || err == ErrFileClosed {
if err == nil || err == ErrFileClosed { //nolint:errorlint // err is Errno
err = ErrPipeListenerClosed
}
}
@ -402,15 +471,15 @@ func (l *win32PipeListener) listenerRoutine() {
p, err = l.makeConnectedServerPipe()
// If the connection was immediately closed by the client, try
// again.
if err != cERROR_NO_DATA {
if err != windows.ERROR_NO_DATA { //nolint:errorlint // err is Errno
break
}
}
responseCh <- acceptResponse{p, err}
closed = err == ErrPipeListenerClosed
closed = err == ErrPipeListenerClosed //nolint:errorlint // err is Errno
}
}
syscall.Close(l.firstHandle)
windows.Close(l.firstHandle)
l.firstHandle = 0
// Notify Close() and Accept() callers that the handle has been closed.
close(l.doneCh)
@ -469,15 +538,15 @@ func ListenPipe(path string, c *PipeConfig) (net.Listener, error) {
}
func connectPipe(p *win32File) error {
c, err := p.prepareIo()
c, err := p.prepareIO()
if err != nil {
return err
}
defer p.wg.Done()
err = connectNamedPipe(p.handle, &c.o)
_, err = p.asyncIo(c, nil, 0, err)
if err != nil && err != cERROR_PIPE_CONNECTED {
_, err = p.asyncIO(c, nil, 0, err)
if err != nil && err != windows.ERROR_PIPE_CONNECTED { //nolint:errorlint // err is Errno
return err
}
return nil

View File

@ -0,0 +1,307 @@
//go:build windows
// +build windows
package bindfilter
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"unsafe"
"golang.org/x/sys/windows"
)
//go:generate go run github.com/Microsoft/go-winio/tools/mkwinsyscall -output zsyscall_windows.go ./bind_filter.go
//sys bfSetupFilter(jobHandle windows.Handle, flags uint32, virtRootPath string, virtTargetPath string, virtExceptions **uint16, virtExceptionPathCount uint32) (hr error) = bindfltapi.BfSetupFilter?
//sys bfRemoveMapping(jobHandle windows.Handle, virtRootPath string) (hr error) = bindfltapi.BfRemoveMapping?
//sys bfGetMappings(flags uint32, jobHandle windows.Handle, virtRootPath *uint16, sid *windows.SID, bufferSize *uint32, outBuffer *byte) (hr error) = bindfltapi.BfGetMappings?
// BfSetupFilter flags. See:
// https://github.com/microsoft/BuildXL/blob/a6dce509f0d4f774255e5fbfb75fa6d5290ed163/Public/Src/Utilities/Native/Processes/Windows/NativeContainerUtilities.cs#L193-L240
//
//nolint:revive // var-naming: ALL_CAPS
const (
BINDFLT_FLAG_READ_ONLY_MAPPING uint32 = 0x00000001
// Tells bindflt to fail mapping with STATUS_INVALID_PARAMETER if a mapping produces
// multiple targets.
BINDFLT_FLAG_NO_MULTIPLE_TARGETS uint32 = 0x00000040
)
//nolint:revive // var-naming: ALL_CAPS
const (
BINDFLT_GET_MAPPINGS_FLAG_VOLUME uint32 = 0x00000001
BINDFLT_GET_MAPPINGS_FLAG_SILO uint32 = 0x00000002
BINDFLT_GET_MAPPINGS_FLAG_USER uint32 = 0x00000004
)
// ApplyFileBinding creates a global mount of the source in root, with an optional
// read only flag.
// The bind filter allows us to create mounts of directories and volumes. By default it allows
// us to mount multiple sources inside a single root, acting as an overlay. Files from the
// second source will superscede the first source that was mounted.
// This function disables this behavior and sets the BINDFLT_FLAG_NO_MULTIPLE_TARGETS flag
// on the mount.
func ApplyFileBinding(root, source string, readOnly bool) error {
// The parent directory needs to exist for the bind to work. MkdirAll stats and
// returns nil if the directory exists internally so we should be fine to mkdirall
// every time.
if err := os.MkdirAll(filepath.Dir(root), 0); err != nil {
return err
}
if strings.Contains(source, "Volume{") && !strings.HasSuffix(source, "\\") {
// Add trailing slash to volumes, otherwise we get an error when binding it to
// a folder.
source = source + "\\"
}
flags := BINDFLT_FLAG_NO_MULTIPLE_TARGETS
if readOnly {
flags |= BINDFLT_FLAG_READ_ONLY_MAPPING
}
// Set the job handle to 0 to create a global mount.
if err := bfSetupFilter(
0,
flags,
root,
source,
nil,
0,
); err != nil {
return fmt.Errorf("failed to bind target %q to root %q: %w", source, root, err)
}
return nil
}
// RemoveFileBinding removes a mount from the root path.
func RemoveFileBinding(root string) error {
if err := bfRemoveMapping(0, root); err != nil {
return fmt.Errorf("removing file binding: %w", err)
}
return nil
}
// GetBindMappings returns a list of bind mappings that have their root on a
// particular volume. The volumePath parameter can be any path that exists on
// a volume. For example, if a number of mappings are created in C:\ProgramData\test,
// to get a list of those mappings, the volumePath parameter would have to be set to
// C:\ or the VOLUME_NAME_GUID notation of C:\ (\\?\Volume{GUID}\), or any child
// path that exists.
func GetBindMappings(volumePath string) ([]BindMapping, error) {
rootPtr, err := windows.UTF16PtrFromString(volumePath)
if err != nil {
return nil, err
}
flags := BINDFLT_GET_MAPPINGS_FLAG_VOLUME
// allocate a large buffer for results
var outBuffSize uint32 = 256 * 1024
buf := make([]byte, outBuffSize)
if err := bfGetMappings(flags, 0, rootPtr, nil, &outBuffSize, &buf[0]); err != nil {
return nil, err
}
if outBuffSize < 12 {
return nil, fmt.Errorf("invalid buffer returned")
}
result := buf[:outBuffSize]
// The first 12 bytes are the three uint32 fields in getMappingsResponseHeader{}
headerBuffer := result[:12]
// The alternative to using unsafe and casting it to the above defined structures, is to manually
// parse the fields. Not too terrible, but not sure it'd worth the trouble.
header := *(*getMappingsResponseHeader)(unsafe.Pointer(&headerBuffer[0]))
if header.MappingCount == 0 {
// no mappings
return []BindMapping{}, nil
}
mappingsBuffer := result[12 : int(unsafe.Sizeof(mappingEntry{}))*int(header.MappingCount)]
// Get a pointer to the first mapping in the slice
mappingsPointer := (*mappingEntry)(unsafe.Pointer(&mappingsBuffer[0]))
// Get slice of mappings
mappings := unsafe.Slice(mappingsPointer, header.MappingCount)
mappingEntries := make([]BindMapping, header.MappingCount)
for i := 0; i < int(header.MappingCount); i++ {
bindMapping, err := getBindMappingFromBuffer(result, mappings[i])
if err != nil {
return nil, fmt.Errorf("fetching bind mappings: %w", err)
}
mappingEntries[i] = bindMapping
}
return mappingEntries, nil
}
// mappingEntry holds information about where in the response buffer we can
// find information about the virtual root (the mount point) and the targets (sources)
// that get mounted, as well as the flags used to bind the targets to the virtual root.
type mappingEntry struct {
VirtRootLength uint32
VirtRootOffset uint32
Flags uint32
NumberOfTargets uint32
TargetEntriesOffset uint32
}
type mappingTargetEntry struct {
TargetRootLength uint32
TargetRootOffset uint32
}
// getMappingsResponseHeader represents the first 12 bytes of the BfGetMappings() response.
// It gives us the size of the buffer, the status of the call and the number of mappings.
// A response
type getMappingsResponseHeader struct {
Size uint32
Status uint32
MappingCount uint32
}
type BindMapping struct {
MountPoint string
Flags uint32
Targets []string
}
func decodeEntry(buffer []byte) (string, error) {
name := make([]uint16, len(buffer)/2)
err := binary.Read(bytes.NewReader(buffer), binary.LittleEndian, &name)
if err != nil {
return "", fmt.Errorf("decoding name: %w", err)
}
return windows.UTF16ToString(name), nil
}
func getTargetsFromBuffer(buffer []byte, offset, count int) ([]string, error) {
if len(buffer) < offset+count*6 {
return nil, fmt.Errorf("invalid buffer")
}
targets := make([]string, count)
for i := 0; i < count; i++ {
entryBuf := buffer[offset+i*8 : offset+i*8+8]
tgt := *(*mappingTargetEntry)(unsafe.Pointer(&entryBuf[0]))
if len(buffer) < int(tgt.TargetRootOffset)+int(tgt.TargetRootLength) {
return nil, fmt.Errorf("invalid buffer")
}
decoded, err := decodeEntry(buffer[tgt.TargetRootOffset : tgt.TargetRootOffset+tgt.TargetRootLength])
if err != nil {
return nil, fmt.Errorf("decoding name: %w", err)
}
decoded, err = getFinalPath(decoded)
if err != nil {
return nil, fmt.Errorf("fetching final path: %w", err)
}
targets[i] = decoded
}
return targets, nil
}
func getFinalPath(pth string) (string, error) {
// BfGetMappings returns VOLUME_NAME_NT paths like \Device\HarddiskVolume2\ProgramData.
// These can be accessed by prepending \\.\GLOBALROOT to the path. We use this to get the
// DOS paths for these files.
if strings.HasPrefix(pth, `\Device`) {
pth = `\\.\GLOBALROOT` + pth
}
han, err := openPath(pth)
if err != nil {
return "", fmt.Errorf("fetching file handle: %w", err)
}
defer func() {
_ = windows.CloseHandle(han)
}()
buf := make([]uint16, 100)
var flags uint32 = 0x0
for {
n, err := windows.GetFinalPathNameByHandle(han, &buf[0], uint32(len(buf)), flags)
if err != nil {
// if we mounted a volume that does not also have a drive letter assigned, attempting to
// fetch the VOLUME_NAME_DOS will fail with os.ErrNotExist. Attempt to get the VOLUME_NAME_GUID.
if errors.Is(err, os.ErrNotExist) && flags != 0x1 {
flags = 0x1
continue
}
return "", fmt.Errorf("getting final path name: %w", err)
}
if n < uint32(len(buf)) {
break
}
buf = make([]uint16, n)
}
finalPath := windows.UTF16ToString(buf)
// We got VOLUME_NAME_DOS, we need to strip away some leading slashes.
// Leave unchanged if we ended up requesting VOLUME_NAME_GUID
if len(finalPath) > 4 && finalPath[:4] == `\\?\` && flags == 0x0 {
finalPath = finalPath[4:]
if len(finalPath) > 3 && finalPath[:3] == `UNC` {
// return path like \\server\share\...
finalPath = `\` + finalPath[3:]
}
}
return finalPath, nil
}
func getBindMappingFromBuffer(buffer []byte, entry mappingEntry) (BindMapping, error) {
if len(buffer) < int(entry.VirtRootOffset)+int(entry.VirtRootLength) {
return BindMapping{}, fmt.Errorf("invalid buffer")
}
src, err := decodeEntry(buffer[entry.VirtRootOffset : entry.VirtRootOffset+entry.VirtRootLength])
if err != nil {
return BindMapping{}, fmt.Errorf("decoding entry: %w", err)
}
targets, err := getTargetsFromBuffer(buffer, int(entry.TargetEntriesOffset), int(entry.NumberOfTargets))
if err != nil {
return BindMapping{}, fmt.Errorf("fetching targets: %w", err)
}
src, err = getFinalPath(src)
if err != nil {
return BindMapping{}, fmt.Errorf("fetching final path: %w", err)
}
return BindMapping{
Flags: entry.Flags,
Targets: targets,
MountPoint: src,
}, nil
}
func openPath(path string) (windows.Handle, error) {
u16, err := windows.UTF16PtrFromString(path)
if err != nil {
return 0, err
}
h, err := windows.CreateFile(
u16,
0,
windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE,
nil,
windows.OPEN_EXISTING,
windows.FILE_FLAG_BACKUP_SEMANTICS, // Needed to open a directory handle.
0)
if err != nil {
return 0, &os.PathError{
Op: "CreateFile",
Path: path,
Err: err,
}
}
return h, nil
}

View File

@ -0,0 +1,113 @@
//go:build windows
// Code generated by 'go generate' using "github.com/Microsoft/go-winio/tools/mkwinsyscall"; DO NOT EDIT.
package bindfilter
import (
"syscall"
"unsafe"
"golang.org/x/sys/windows"
)
var _ unsafe.Pointer
// Do the interface allocations only once for common
// Errno values.
const (
errnoERROR_IO_PENDING = 997
)
var (
errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING)
errERROR_EINVAL error = syscall.EINVAL
)
// errnoErr returns common boxed Errno values, to prevent
// allocations at runtime.
func errnoErr(e syscall.Errno) error {
switch e {
case 0:
return errERROR_EINVAL
case errnoERROR_IO_PENDING:
return errERROR_IO_PENDING
}
return e
}
var (
modbindfltapi = windows.NewLazySystemDLL("bindfltapi.dll")
procBfGetMappings = modbindfltapi.NewProc("BfGetMappings")
procBfRemoveMapping = modbindfltapi.NewProc("BfRemoveMapping")
procBfSetupFilter = modbindfltapi.NewProc("BfSetupFilter")
)
func bfGetMappings(flags uint32, jobHandle windows.Handle, virtRootPath *uint16, sid *windows.SID, bufferSize *uint32, outBuffer *byte) (hr error) {
hr = procBfGetMappings.Find()
if hr != nil {
return
}
r0, _, _ := syscall.SyscallN(procBfGetMappings.Addr(), uintptr(flags), uintptr(jobHandle), uintptr(unsafe.Pointer(virtRootPath)), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(bufferSize)), uintptr(unsafe.Pointer(outBuffer)))
if int32(r0) < 0 {
if r0&0x1fff0000 == 0x00070000 {
r0 &= 0xffff
}
hr = syscall.Errno(r0)
}
return
}
func bfRemoveMapping(jobHandle windows.Handle, virtRootPath string) (hr error) {
var _p0 *uint16
_p0, hr = syscall.UTF16PtrFromString(virtRootPath)
if hr != nil {
return
}
return _bfRemoveMapping(jobHandle, _p0)
}
func _bfRemoveMapping(jobHandle windows.Handle, virtRootPath *uint16) (hr error) {
hr = procBfRemoveMapping.Find()
if hr != nil {
return
}
r0, _, _ := syscall.SyscallN(procBfRemoveMapping.Addr(), uintptr(jobHandle), uintptr(unsafe.Pointer(virtRootPath)))
if int32(r0) < 0 {
if r0&0x1fff0000 == 0x00070000 {
r0 &= 0xffff
}
hr = syscall.Errno(r0)
}
return
}
func bfSetupFilter(jobHandle windows.Handle, flags uint32, virtRootPath string, virtTargetPath string, virtExceptions **uint16, virtExceptionPathCount uint32) (hr error) {
var _p0 *uint16
_p0, hr = syscall.UTF16PtrFromString(virtRootPath)
if hr != nil {
return
}
var _p1 *uint16
_p1, hr = syscall.UTF16PtrFromString(virtTargetPath)
if hr != nil {
return
}
return _bfSetupFilter(jobHandle, flags, _p0, _p1, virtExceptions, virtExceptionPathCount)
}
func _bfSetupFilter(jobHandle windows.Handle, flags uint32, virtRootPath *uint16, virtTargetPath *uint16, virtExceptions **uint16, virtExceptionPathCount uint32) (hr error) {
hr = procBfSetupFilter.Find()
if hr != nil {
return
}
r0, _, _ := syscall.SyscallN(procBfSetupFilter.Addr(), uintptr(jobHandle), uintptr(flags), uintptr(unsafe.Pointer(virtRootPath)), uintptr(unsafe.Pointer(virtTargetPath)), uintptr(unsafe.Pointer(virtExceptions)), uintptr(virtExceptionPathCount))
if int32(r0) < 0 {
if r0&0x1fff0000 == 0x00070000 {
r0 &= 0xffff
}
hr = syscall.Errno(r0)
}
return
}

View File

@ -1,5 +1,3 @@
// +build windows
// Package guid provides a GUID type. The backing structure for a GUID is
// identical to that used by the golang.org/x/sys/windows GUID type.
// There are two main binary encodings used for a GUID, the big-endian encoding,
@ -9,26 +7,26 @@ package guid
import (
"crypto/rand"
"crypto/sha1"
"crypto/sha1" //nolint:gosec // not used for secure application
"encoding"
"encoding/binary"
"fmt"
"strconv"
"golang.org/x/sys/windows"
)
//go:generate go run golang.org/x/tools/cmd/stringer -type=Variant -trimprefix=Variant -linecomment
// Variant specifies which GUID variant (or "type") of the GUID. It determines
// how the entirety of the rest of the GUID is interpreted.
type Variant uint8
// The variants specified by RFC 4122.
// The variants specified by RFC 4122 section 4.1.1.
const (
// VariantUnknown specifies a GUID variant which does not conform to one of
// the variant encodings specified in RFC 4122.
VariantUnknown Variant = iota
VariantNCS
VariantRFC4122
VariantRFC4122 // RFC 4122
VariantMicrosoft
VariantFuture
)
@ -38,16 +36,13 @@ const (
// hash of an input string.
type Version uint8
func (v Version) String() string {
return strconv.FormatUint(uint64(v), 10)
}
var _ = (encoding.TextMarshaler)(GUID{})
var _ = (encoding.TextUnmarshaler)(&GUID{})
// GUID represents a GUID/UUID. It has the same structure as
// golang.org/x/sys/windows.GUID so that it can be used with functions expecting
// that type. It is defined as its own type so that stringification and
// marshaling can be supported. The representation matches that used by native
// Windows code.
type GUID windows.GUID
// NewV4 returns a new version 4 (pseudorandom) GUID, as defined by RFC 4122.
func NewV4() (GUID, error) {
var b [16]byte
@ -70,7 +65,7 @@ func NewV4() (GUID, error) {
// big-endian UTF16 stream of bytes. If that is desired, the string can be
// encoded as such before being passed to this function.
func NewV5(namespace GUID, name []byte) (GUID, error) {
b := sha1.New()
b := sha1.New() //nolint:gosec // not used for secure application
namespaceBytes := namespace.ToArray()
b.Write(namespaceBytes[:])
b.Write(name)

View File

@ -0,0 +1,16 @@
//go:build !windows
// +build !windows
package guid
// GUID represents a GUID/UUID. It has the same structure as
// golang.org/x/sys/windows.GUID so that it can be used with functions expecting
// that type. It is defined as its own type as that is only available to builds
// targeted at `windows`. The representation matches that used by native Windows
// code.
type GUID struct {
Data1 uint32
Data2 uint16
Data3 uint16
Data4 [8]byte
}

View File

@ -0,0 +1,13 @@
//go:build windows
// +build windows
package guid
import "golang.org/x/sys/windows"
// GUID represents a GUID/UUID. It has the same structure as
// golang.org/x/sys/windows.GUID so that it can be used with functions expecting
// that type. It is defined as its own type so that stringification and
// marshaling can be supported. The representation matches that used by native
// Windows code.
type GUID windows.GUID

View File

@ -0,0 +1,27 @@
// Code generated by "stringer -type=Variant -trimprefix=Variant -linecomment"; DO NOT EDIT.
package guid
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[VariantUnknown-0]
_ = x[VariantNCS-1]
_ = x[VariantRFC4122-2]
_ = x[VariantMicrosoft-3]
_ = x[VariantFuture-4]
}
const _Variant_name = "UnknownNCSRFC 4122MicrosoftFuture"
var _Variant_index = [...]uint8{0, 7, 10, 18, 27, 33}
func (i Variant) String() string {
if i >= Variant(len(_Variant_index)-1) {
return "Variant(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _Variant_name[_Variant_index[i]:_Variant_index[i+1]]
}

View File

@ -1,3 +1,4 @@
//go:build windows
// +build windows
package winio
@ -8,7 +9,6 @@ import (
"fmt"
"runtime"
"sync"
"syscall"
"unicode/utf16"
"golang.org/x/sys/windows"
@ -17,29 +17,24 @@ import (
//sys adjustTokenPrivileges(token windows.Token, releaseAll bool, input *byte, outputSize uint32, output *byte, requiredSize *uint32) (success bool, err error) [true] = advapi32.AdjustTokenPrivileges
//sys impersonateSelf(level uint32) (err error) = advapi32.ImpersonateSelf
//sys revertToSelf() (err error) = advapi32.RevertToSelf
//sys openThreadToken(thread syscall.Handle, accessMask uint32, openAsSelf bool, token *windows.Token) (err error) = advapi32.OpenThreadToken
//sys getCurrentThread() (h syscall.Handle) = GetCurrentThread
//sys openThreadToken(thread windows.Handle, accessMask uint32, openAsSelf bool, token *windows.Token) (err error) = advapi32.OpenThreadToken
//sys getCurrentThread() (h windows.Handle) = GetCurrentThread
//sys lookupPrivilegeValue(systemName string, name string, luid *uint64) (err error) = advapi32.LookupPrivilegeValueW
//sys lookupPrivilegeName(systemName string, luid *uint64, buffer *uint16, size *uint32) (err error) = advapi32.LookupPrivilegeNameW
//sys lookupPrivilegeDisplayName(systemName string, name *uint16, buffer *uint16, size *uint32, languageId *uint32) (err error) = advapi32.LookupPrivilegeDisplayNameW
const (
SE_PRIVILEGE_ENABLED = 2
//revive:disable-next-line:var-naming ALL_CAPS
SE_PRIVILEGE_ENABLED = windows.SE_PRIVILEGE_ENABLED
ERROR_NOT_ALL_ASSIGNED syscall.Errno = 1300
//revive:disable-next-line:var-naming ALL_CAPS
ERROR_NOT_ALL_ASSIGNED windows.Errno = windows.ERROR_NOT_ALL_ASSIGNED
SeBackupPrivilege = "SeBackupPrivilege"
SeRestorePrivilege = "SeRestorePrivilege"
SeSecurityPrivilege = "SeSecurityPrivilege"
)
const (
securityAnonymous = iota
securityIdentification
securityImpersonation
securityDelegation
)
var (
privNames = make(map[string]uint64)
privNameMutex sync.Mutex
@ -51,11 +46,9 @@ type PrivilegeError struct {
}
func (e *PrivilegeError) Error() string {
s := ""
s := "Could not enable privilege "
if len(e.privileges) > 1 {
s = "Could not enable privileges "
} else {
s = "Could not enable privilege "
}
for i, p := range e.privileges {
if i != 0 {
@ -94,7 +87,7 @@ func RunWithPrivileges(names []string, fn func() error) error {
}
func mapPrivileges(names []string) ([]uint64, error) {
var privileges []uint64
privileges := make([]uint64, 0, len(names))
privNameMutex.Lock()
defer privNameMutex.Unlock()
for _, name := range names {
@ -127,7 +120,7 @@ func enableDisableProcessPrivilege(names []string, action uint32) error {
return err
}
p, _ := windows.GetCurrentProcess()
p := windows.CurrentProcess()
var token windows.Token
err = windows.OpenProcessToken(p, windows.TOKEN_ADJUST_PRIVILEGES|windows.TOKEN_QUERY, &token)
if err != nil {
@ -140,10 +133,10 @@ func enableDisableProcessPrivilege(names []string, action uint32) error {
func adjustPrivileges(token windows.Token, privileges []uint64, action uint32) error {
var b bytes.Buffer
binary.Write(&b, binary.LittleEndian, uint32(len(privileges)))
_ = binary.Write(&b, binary.LittleEndian, uint32(len(privileges)))
for _, p := range privileges {
binary.Write(&b, binary.LittleEndian, p)
binary.Write(&b, binary.LittleEndian, action)
_ = binary.Write(&b, binary.LittleEndian, p)
_ = binary.Write(&b, binary.LittleEndian, action)
}
prevState := make([]byte, b.Len())
reqSize := uint32(0)
@ -151,7 +144,7 @@ func adjustPrivileges(token windows.Token, privileges []uint64, action uint32) e
if !success {
return err
}
if err == ERROR_NOT_ALL_ASSIGNED {
if err == ERROR_NOT_ALL_ASSIGNED { //nolint:errorlint // err is Errno
return &PrivilegeError{privileges}
}
return nil
@ -177,13 +170,13 @@ func getPrivilegeName(luid uint64) string {
}
func newThreadToken() (windows.Token, error) {
err := impersonateSelf(securityImpersonation)
err := impersonateSelf(windows.SecurityImpersonation)
if err != nil {
return 0, err
}
var token windows.Token
err = openThreadToken(getCurrentThread(), syscall.TOKEN_ADJUST_PRIVILEGES|syscall.TOKEN_QUERY, false, &token)
err = openThreadToken(getCurrentThread(), windows.TOKEN_ADJUST_PRIVILEGES|windows.TOKEN_QUERY, false, &token)
if err != nil {
rerr := revertToSelf()
if rerr != nil {

Some files were not shown because too many files have changed in this diff Show More