This commit is contained in:
2026-02-19 10:07:43 +00:00
parent 007438e372
commit 6e637ecf77
1763 changed files with 60820 additions and 279516 deletions

View File

@@ -1,255 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build linux && !ts_omit_aws
// Package awsstore contains an ipn.StateStore implementation using AWS SSM.
package awsstore
import (
"context"
"errors"
"fmt"
"net/url"
"regexp"
"strings"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/aws/arn"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/ssm"
ssmTypes "github.com/aws/aws-sdk-go-v2/service/ssm/types"
"tailscale.com/ipn"
"tailscale.com/ipn/store/mem"
"tailscale.com/types/logger"
)
const (
parameterNameRxStr = `^parameter(/.*)`
)
var parameterNameRx = regexp.MustCompile(parameterNameRxStr)
// Option defines a functional option type for configuring awsStore.
type Option func(*storeOptions)
// storeOptions holds optional settings for creating a new awsStore.
type storeOptions struct {
kmsKey string
}
// awsSSMClient is an interface allowing us to mock the couple of
// API calls we are leveraging with the AWSStore provider
type awsSSMClient interface {
GetParameter(ctx context.Context,
params *ssm.GetParameterInput,
optFns ...func(*ssm.Options)) (*ssm.GetParameterOutput, error)
PutParameter(ctx context.Context,
params *ssm.PutParameterInput,
optFns ...func(*ssm.Options)) (*ssm.PutParameterOutput, error)
}
// store is a store which leverages AWS SSM parameter store
// to persist the state
type awsStore struct {
ssmClient awsSSMClient
ssmARN arn.ARN
// kmsKey is optional. If empty, the parameter is stored in plaintext.
// If non-empty, the parameter is encrypted with this KMS key.
kmsKey string
memory mem.Store
}
// New returns a new ipn.StateStore using the AWS SSM storage
// location given by ssmARN.
//
// Note that we store the entire store in a single parameter
// key, therefore if the state is above 8kb, it can cause
// Tailscaled to only only store new state in-memory and
// restarting Tailscaled can fail until you delete your state
// from the AWS Parameter Store.
//
// If you want to specify an optional KMS key,
// pass one or more Option objects, e.g. awsstore.WithKeyID("alias/my-key").
func New(_ logger.Logf, ssmARN string, opts ...Option) (ipn.StateStore, error) {
// Apply all options to an empty storeOptions
var so storeOptions
for _, opt := range opts {
opt(&so)
}
return newStore(ssmARN, so, nil)
}
// WithKeyID sets the KMS key to be used for encryption. It can be
// a KeyID, an alias ("alias/my-key"), or a full ARN.
//
// If kmsKey is empty, the Option is a no-op.
func WithKeyID(kmsKey string) Option {
return func(o *storeOptions) {
o.kmsKey = kmsKey
}
}
// ParseARNAndOpts parses an ARN and optional URL-encoded parameters
// from arg.
func ParseARNAndOpts(arg string) (ssmARN string, opts []Option, err error) {
ssmARN = arg
// Support optional ?url-encoded-parameters.
if s, q, ok := strings.Cut(arg, "?"); ok {
ssmARN = s
q, err := url.ParseQuery(q)
if err != nil {
return "", nil, err
}
for k := range q {
switch k {
default:
return "", nil, fmt.Errorf("unknown arn option parameter %q", k)
case "kmsKey":
// We allow an ARN, a key ID, or an alias name for kmsKeyID.
// If it doesn't look like an ARN and doesn't have a '/',
// prepend "alias/" for KMS alias references.
kmsKey := q.Get(k)
if kmsKey != "" &&
!strings.Contains(kmsKey, "/") &&
!strings.HasPrefix(kmsKey, "arn:") {
kmsKey = "alias/" + kmsKey
}
if kmsKey != "" {
opts = append(opts, WithKeyID(kmsKey))
}
}
}
}
return ssmARN, opts, nil
}
// newStore is NewStore, but for tests. If client is non-nil, it's
// used instead of making one.
func newStore(ssmARN string, so storeOptions, client awsSSMClient) (ipn.StateStore, error) {
s := &awsStore{
ssmClient: client,
kmsKey: so.kmsKey,
}
var err error
if s.ssmARN, err = arn.Parse(ssmARN); err != nil {
return nil, fmt.Errorf("unable to parse the ARN correctly: %v", err)
}
if s.ssmARN.Service != "ssm" {
return nil, fmt.Errorf("invalid service %q, expected 'ssm'", s.ssmARN.Service)
}
if !parameterNameRx.MatchString(s.ssmARN.Resource) {
return nil, fmt.Errorf("invalid resource %q, expected to match %v", s.ssmARN.Resource, parameterNameRxStr)
}
if s.ssmClient == nil {
var cfg aws.Config
if cfg, err = config.LoadDefaultConfig(
context.TODO(),
config.WithRegion(s.ssmARN.Region),
); err != nil {
return nil, err
}
s.ssmClient = ssm.NewFromConfig(cfg)
}
// Preload existing state, if any
if err := s.LoadState(); err != nil {
return nil, err
}
return s, nil
}
// LoadState attempts to read the state from AWS SSM parameter store key.
func (s *awsStore) LoadState() error {
param, err := s.ssmClient.GetParameter(
context.TODO(),
&ssm.GetParameterInput{
Name: aws.String(s.ParameterName()),
WithDecryption: aws.Bool(true),
},
)
if err != nil {
var pnf *ssmTypes.ParameterNotFound
if errors.As(err, &pnf) {
// Create the parameter as it does not exist yet
// and return directly as it is defacto empty
return s.persistState()
}
return err
}
// Load the content in-memory
return s.memory.LoadFromJSON([]byte(*param.Parameter.Value))
}
// ParameterName returns the parameter name extracted from
// the provided ARN
func (s *awsStore) ParameterName() (name string) {
values := parameterNameRx.FindStringSubmatch(s.ssmARN.Resource)
if len(values) == 2 {
name = values[1]
}
return
}
// String returns the awsStore and the ARN of the SSM parameter store
// configured to store the state
func (s *awsStore) String() string { return fmt.Sprintf("awsStore(%q)", s.ssmARN.String()) }
// ReadState implements the Store interface.
func (s *awsStore) ReadState(id ipn.StateKey) (bs []byte, err error) {
return s.memory.ReadState(id)
}
// WriteState implements the Store interface.
func (s *awsStore) WriteState(id ipn.StateKey, bs []byte) (err error) {
// Write the state in-memory
if err = s.memory.WriteState(id, bs); err != nil {
return
}
// Persist the state in AWS SSM parameter store
return s.persistState()
}
// PersistState saves the states into the AWS SSM parameter store
func (s *awsStore) persistState() error {
// Generate JSON from in-memory cache
bs, err := s.memory.ExportToJSON()
if err != nil {
return err
}
// Store in AWS SSM parameter store.
//
// We use intelligent tiering so that when the state is below 4kb, it uses Standard tiering
// which is free. However, if it exceeds 4kb it switches the parameter to advanced tiering
// doubling the capacity to 8kb per the following docs:
// https://aws.amazon.com/about-aws/whats-new/2019/08/aws-systems-manager-parameter-store-announces-intelligent-tiering-to-enable-automatic-parameter-tier-selection/
in := &ssm.PutParameterInput{
Name: aws.String(s.ParameterName()),
Value: aws.String(string(bs)),
Overwrite: aws.Bool(true),
Tier: ssmTypes.ParameterTierIntelligentTiering,
Type: ssmTypes.ParameterTypeSecureString,
}
// If kmsKey is specified, encrypt with that key
// NOTE: this input allows any alias, keyID or ARN
// If this isn't specified, AWS will use the default KMS key
if s.kmsKey != "" {
in.KeyId = aws.String(s.kmsKey)
}
_, err = s.ssmClient.PutParameter(context.TODO(), in)
return err
}

View File

@@ -1,430 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
// Package kubestore contains an ipn.StateStore implementation using Kubernetes Secrets.
package kubestore
import (
"context"
"fmt"
"log"
"net"
"os"
"strings"
"time"
"tailscale.com/envknob"
"tailscale.com/ipn"
"tailscale.com/ipn/store/mem"
"tailscale.com/kube/kubeapi"
"tailscale.com/kube/kubeclient"
"tailscale.com/kube/kubetypes"
"tailscale.com/types/logger"
"tailscale.com/util/dnsname"
"tailscale.com/util/mak"
)
const (
// timeout is the timeout for a single state update that includes calls to the API server to write or read a
// state Secret and emit an Event.
timeout = 30 * time.Second
reasonTailscaleStateUpdated = "TailscaledStateUpdated"
reasonTailscaleStateLoaded = "TailscaleStateLoaded"
reasonTailscaleStateUpdateFailed = "TailscaleStateUpdateFailed"
reasonTailscaleStateLoadFailed = "TailscaleStateLoadFailed"
eventTypeWarning = "Warning"
eventTypeNormal = "Normal"
keyTLSCert = "tls.crt"
keyTLSKey = "tls.key"
)
// Store is an ipn.StateStore that uses a Kubernetes Secret for persistence.
type Store struct {
client kubeclient.Client
canPatch bool
secretName string // state Secret
certShareMode string // 'ro', 'rw', or empty
podName string
// memory holds the latest tailscale state. Writes write state to a kube
// Secret and memory, Reads read from memory.
memory mem.Store
}
// New returns a new Store that persists state to Kubernets Secret(s).
// Tailscale state is stored in a Secret named by the secretName parameter.
// TLS certs are stored and retrieved from state Secret or separate Secrets
// named after TLS endpoints if running in cert share mode.
func New(logf logger.Logf, secretName string) (*Store, error) {
c, err := newClient()
if err != nil {
return nil, err
}
return newWithClient(logf, c, secretName)
}
func newClient() (kubeclient.Client, error) {
c, err := kubeclient.New("tailscale-state-store")
if err != nil {
return nil, err
}
if os.Getenv("TS_KUBERNETES_READ_API_SERVER_ADDRESS_FROM_ENV") == "true" {
// Derive the API server address from the environment variables
c.SetURL(fmt.Sprintf("https://%s:%s", os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")))
}
return c, nil
}
func newWithClient(logf logger.Logf, c kubeclient.Client, secretName string) (*Store, error) {
canPatch, _, err := c.CheckSecretPermissions(context.Background(), secretName)
if err != nil {
return nil, err
}
s := &Store{
client: c,
canPatch: canPatch,
secretName: secretName,
podName: os.Getenv("POD_NAME"),
}
if envknob.IsCertShareReadWriteMode() {
s.certShareMode = "rw"
} else if envknob.IsCertShareReadOnlyMode() {
s.certShareMode = "ro"
}
// Load latest state from kube Secret if it already exists.
if err := s.loadState(); err != nil && err != ipn.ErrStateNotExist {
return nil, fmt.Errorf("error loading state from kube Secret: %w", err)
}
// If we are in cert share mode, pre-load existing shared certs.
if s.certShareMode == "rw" || s.certShareMode == "ro" {
sel := s.certSecretSelector()
if err := s.loadCerts(context.Background(), sel); err != nil {
// We will attempt to again retrieve the certs from Secrets when a request for an HTTPS endpoint
// is received.
log.Printf("[unexpected] error loading TLS certs: %v", err)
}
}
if s.certShareMode == "ro" {
go s.runCertReload(context.Background(), logf)
}
return s, nil
}
func (s *Store) SetDialer(d func(ctx context.Context, network, address string) (net.Conn, error)) {
s.client.SetDialer(d)
}
func (s *Store) String() string { return "kube.Store" }
// ReadState implements the StateStore interface.
func (s *Store) ReadState(id ipn.StateKey) ([]byte, error) {
return s.memory.ReadState(ipn.StateKey(sanitizeKey(id)))
}
// WriteState implements the StateStore interface.
func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) {
defer func() {
if err == nil {
s.memory.WriteState(ipn.StateKey(sanitizeKey(id)), bs)
}
}()
return s.updateSecret(map[string][]byte{string(id): bs}, s.secretName)
}
// WriteTLSCertAndKey writes a TLS cert and key to domain.crt, domain.key fields
// of a Tailscale Kubernetes node's state Secret.
func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error) {
if s.certShareMode == "ro" {
log.Printf("[unexpected] TLS cert and key write in read-only mode")
}
if err := dnsname.ValidHostname(domain); err != nil {
return fmt.Errorf("invalid domain name %q: %w", domain, err)
}
secretName := s.secretName
data := map[string][]byte{
domain + ".crt": cert,
domain + ".key": key,
}
// If we run in cert share mode, cert and key for a DNS name are written
// to a separate Secret.
if s.certShareMode == "rw" {
secretName = domain
data = map[string][]byte{
keyTLSCert: cert,
keyTLSKey: key,
}
}
if err := s.updateSecret(data, secretName); err != nil {
return fmt.Errorf("error writing TLS cert and key to Secret: %w", err)
}
// TODO(irbekrm): certs for write replicas are currently not
// written to memory to avoid out of sync memory state after
// Ingress resources have been recreated. This means that TLS
// certs for write replicas are retrieved from the Secret on
// each HTTPS request. This is a temporary solution till we
// implement a Secret watch.
if s.certShareMode != "rw" {
s.memory.WriteState(ipn.StateKey(domain+".crt"), cert)
s.memory.WriteState(ipn.StateKey(domain+".key"), key)
}
return nil
}
// ReadTLSCertAndKey reads a TLS cert and key from memory or from a
// domain-specific Secret. It first checks the in-memory store, if not found in
// memory and running cert store in read-only mode, looks up a Secret.
// Note that write replicas of HA Ingress always retrieve TLS certs from Secrets.
func (s *Store) ReadTLSCertAndKey(domain string) (cert, key []byte, err error) {
if err := dnsname.ValidHostname(domain); err != nil {
return nil, nil, fmt.Errorf("invalid domain name %q: %w", domain, err)
}
certKey := domain + ".crt"
keyKey := domain + ".key"
cert, err = s.memory.ReadState(ipn.StateKey(certKey))
if err == nil {
key, err = s.memory.ReadState(ipn.StateKey(keyKey))
if err == nil {
return cert, key, nil
}
}
if s.certShareMode == "" {
return nil, nil, ipn.ErrStateNotExist
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
secret, err := s.client.GetSecret(ctx, domain)
if err != nil {
if kubeclient.IsNotFoundErr(err) {
// TODO(irbekrm): we should return a more specific error
// that wraps ipn.ErrStateNotExist here.
return nil, nil, ipn.ErrStateNotExist
}
return nil, nil, fmt.Errorf("getting TLS Secret %q: %w", domain, err)
}
cert = secret.Data[keyTLSCert]
key = secret.Data[keyTLSKey]
if len(cert) == 0 || len(key) == 0 {
return nil, nil, ipn.ErrStateNotExist
}
// TODO(irbekrm): a read between these two separate writes would
// get a mismatched cert and key. Allow writing both cert and
// key to the memory store in a single, lock-protected operation.
//
// TODO(irbekrm): currently certs for write replicas of HA Ingress get
// retrieved from the cluster Secret on each HTTPS request to avoid a
// situation when after Ingress recreation stale certs are read from
// memory.
// Fix this by watching Secrets to ensure that memory store gets updated
// when Secrets are deleted.
if s.certShareMode == "ro" {
s.memory.WriteState(ipn.StateKey(certKey), cert)
s.memory.WriteState(ipn.StateKey(keyKey), key)
}
return cert, key, nil
}
func (s *Store) updateSecret(data map[string][]byte, secretName string) (err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer func() {
if err != nil {
if err := s.client.Event(ctx, eventTypeWarning, reasonTailscaleStateUpdateFailed, err.Error()); err != nil {
log.Printf("kubestore: error creating tailscaled state update Event: %v", err)
}
} else {
if err := s.client.Event(ctx, eventTypeNormal, reasonTailscaleStateUpdated, "Successfully updated tailscaled state Secret"); err != nil {
log.Printf("kubestore: error creating tailscaled state Event: %v", err)
}
}
cancel()
}()
secret, err := s.client.GetSecret(ctx, secretName)
if err != nil {
// If the Secret does not exist, create it with the required data.
if kubeclient.IsNotFoundErr(err) && s.canCreateSecret(secretName) {
return s.client.CreateSecret(ctx, &kubeapi.Secret{
TypeMeta: kubeapi.TypeMeta{
APIVersion: "v1",
Kind: "Secret",
},
ObjectMeta: kubeapi.ObjectMeta{
Name: secretName,
},
Data: func(m map[string][]byte) map[string][]byte {
d := make(map[string][]byte, len(m))
for key, val := range m {
d[sanitizeKey(key)] = val
}
return d
}(data),
})
}
return fmt.Errorf("error getting Secret %s: %w", secretName, err)
}
if s.canPatchSecret(secretName) {
var m []kubeclient.JSONPatch
// If the user has pre-created a Secret with no data, we need to ensure the top level /data field.
if len(secret.Data) == 0 {
m = []kubeclient.JSONPatch{
{
Op: "add",
Path: "/data",
Value: func(m map[string][]byte) map[string][]byte {
d := make(map[string][]byte, len(m))
for key, val := range m {
d[sanitizeKey(key)] = val
}
return d
}(data),
},
}
// If the Secret has data, patch it with the new data.
} else {
for key, val := range data {
m = append(m, kubeclient.JSONPatch{
Op: "add",
Path: "/data/" + sanitizeKey(key),
Value: val,
})
}
}
if err := s.client.JSONPatchResource(ctx, secretName, kubeclient.TypeSecrets, m); err != nil {
return fmt.Errorf("error patching Secret %s: %w", secretName, err)
}
return nil
}
// No patch permissions, use UPDATE instead.
for key, val := range data {
mak.Set(&secret.Data, sanitizeKey(key), val)
}
if err := s.client.UpdateSecret(ctx, secret); err != nil {
return fmt.Errorf("error updating Secret %s: %w", s.secretName, err)
}
return nil
}
func (s *Store) loadState() (err error) {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
secret, err := s.client.GetSecret(ctx, s.secretName)
if err != nil {
if st, ok := err.(*kubeapi.Status); ok && st.Code == 404 {
return ipn.ErrStateNotExist
}
if err := s.client.Event(ctx, eventTypeWarning, reasonTailscaleStateLoadFailed, err.Error()); err != nil {
log.Printf("kubestore: error creating Event: %v", err)
}
return err
}
if err := s.client.Event(ctx, eventTypeNormal, reasonTailscaleStateLoaded, "Successfully loaded tailscaled state from Secret"); err != nil {
log.Printf("kubestore: error creating Event: %v", err)
}
s.memory.LoadFromMap(secret.Data)
return nil
}
// runCertReload relists and reloads all TLS certs for endpoints shared by this
// node from Secrets other than the state Secret to ensure that renewed certs get eventually loaded.
// It is not critical to reload a cert immediately after
// renewal, so a daily check is acceptable.
// Currently (3/2025) this is only used for the shared HA Ingress certs on 'read' replicas.
// Note that if shared certs are not found in memory on an HTTPS request, we
// do a Secret lookup, so this mechanism does not need to ensure that newly
// added Ingresses' certs get loaded.
func (s *Store) runCertReload(ctx context.Context, logf logger.Logf) {
ticker := time.NewTicker(time.Hour * 24)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
sel := s.certSecretSelector()
if err := s.loadCerts(ctx, sel); err != nil {
logf("[unexpected] error reloading TLS certs: %v", err)
}
}
}
}
// loadCerts lists all Secrets matching the provided selector and loads TLS
// certs and keys from those.
func (s *Store) loadCerts(ctx context.Context, sel map[string]string) error {
ss, err := s.client.ListSecrets(ctx, sel)
if err != nil {
return fmt.Errorf("error listing TLS Secrets: %w", err)
}
for _, secret := range ss.Items {
if !hasTLSData(&secret) {
continue
}
// Only load secrets that have valid domain names (ending in .ts.net)
if !strings.HasSuffix(secret.Name, ".ts.net") {
continue
}
s.memory.WriteState(ipn.StateKey(secret.Name)+".crt", secret.Data[keyTLSCert])
s.memory.WriteState(ipn.StateKey(secret.Name)+".key", secret.Data[keyTLSKey])
}
return nil
}
// canCreateSecret returns true if this node should be allowed to create the given
// Secret in its namespace.
func (s *Store) canCreateSecret(secret string) bool {
// Only allow creating the state Secret (and not TLS Secrets).
return secret == s.secretName
}
// canPatchSecret returns true if this node should be allowed to patch the given
// Secret.
func (s *Store) canPatchSecret(secret string) bool {
// For backwards compatibility reasons, setups where the proxies are not
// given PATCH permissions for state Secrets are allowed. For TLS
// Secrets, we should always have PATCH permissions.
if secret == s.secretName {
return s.canPatch
}
return true
}
// certSecretSelector returns a label selector that can be used to list all
// Secrets that aren't Tailscale state Secrets and contain TLS certificates for
// HTTPS endpoints that this node serves.
// Currently (3/2025) this only applies to the Kubernetes Operator's ingress
// ProxyGroup.
func (s *Store) certSecretSelector() map[string]string {
if s.podName == "" {
return map[string]string{}
}
p := strings.LastIndex(s.podName, "-")
if p == -1 {
return map[string]string{}
}
pgName := s.podName[:p]
return map[string]string{
kubetypes.LabelSecretType: "certs",
kubetypes.LabelManaged: "true",
"tailscale.com/proxy-group": pgName,
}
}
// hasTLSData returns true if the provided Secret contains non-empty TLS cert and key.
func hasTLSData(s *kubeapi.Secret) bool {
return len(s.Data[keyTLSCert]) != 0 && len(s.Data[keyTLSKey]) != 0
}
// sanitizeKey converts any value that can be converted to a string into a valid Kubernetes Secret key.
// Valid characters are alphanumeric, -, _, and .
// https://kubernetes.io/docs/concepts/configuration/secret/#restriction-names-data.
func sanitizeKey[T ~string](k T) string {
return strings.Map(func(r rune) rune {
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' || r == '.' {
return r
}
return '_'
}, string(k))
}

View File

@@ -1,26 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build (ts_aws || (linux && (arm64 || amd64))) && !ts_omit_aws
package store
import (
"tailscale.com/ipn"
"tailscale.com/ipn/store/awsstore"
"tailscale.com/types/logger"
)
func init() {
registerAvailableExternalStores = append(registerAvailableExternalStores, registerAWSStore)
}
func registerAWSStore() {
Register("arn:", func(logf logger.Logf, arg string) (ipn.StateStore, error) {
ssmARN, opts, err := awsstore.ParseARNAndOpts(arg)
if err != nil {
return nil, err
}
return awsstore.New(logf, ssmARN, opts...)
})
}

View File

@@ -1,25 +0,0 @@
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
//go:build (ts_kube || (linux && (arm64 || amd64))) && !ts_omit_kube
package store
import (
"strings"
"tailscale.com/ipn"
"tailscale.com/ipn/store/kubestore"
"tailscale.com/types/logger"
)
func init() {
registerAvailableExternalStores = append(registerAvailableExternalStores, registerKubeStore)
}
func registerKubeStore() {
Register("kube:", func(logf logger.Logf, path string) (ipn.StateStore, error) {
secretName := strings.TrimPrefix(path, "kube:")
return kubestore.New(logf, secretName)
})
}

View File

@@ -7,10 +7,14 @@ package store
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"iter"
"maps"
"os"
"path/filepath"
"runtime"
"slices"
"strings"
"sync"
@@ -20,26 +24,22 @@ import (
"tailscale.com/paths"
"tailscale.com/types/logger"
"tailscale.com/util/mak"
"tailscale.com/util/testenv"
)
// Provider returns a StateStore for the provided path.
// The arg is of the form "prefix:rest", where prefix was previously registered with Register.
type Provider func(logf logger.Logf, arg string) (ipn.StateStore, error)
var regOnce sync.Once
var registerAvailableExternalStores []func()
func registerDefaultStores() {
func init() {
Register("mem:", mem.New)
for _, f := range registerAvailableExternalStores {
f()
}
}
var knownStores map[string]Provider
// TPMPrefix is the path prefix used for TPM-encrypted StateStore.
const TPMPrefix = "tpmseal:"
// New returns a StateStore based on the provided arg
// and registered stores.
// The arg is of the form "prefix:rest", where prefix was previously
@@ -53,19 +53,31 @@ var knownStores map[string]Provider
// the suffix an AWS ARN for an SSM.
// - (Linux-only) if the string begins with "kube:",
// the suffix is a Kubernetes secret name
// - (Linux or Windows) if the string begins with "tpmseal:", the suffix is
// filepath that is sealed with the local TPM device.
// - In all other cases, the path is treated as a filepath.
func New(logf logger.Logf, path string) (ipn.StateStore, error) {
regOnce.Do(registerDefaultStores)
for prefix, sf := range knownStores {
if strings.HasPrefix(path, prefix) {
// We can't strip the prefix here as some NewStoreFunc (like arn:)
// expect the prefix.
if prefix == TPMPrefix {
if runtime.GOOS == "windows" {
path = TPMPrefix + TryWindowsAppDataMigration(logf, strings.TrimPrefix(path, TPMPrefix))
}
if err := maybeMigrateLocalStateFile(logf, path); err != nil {
return nil, fmt.Errorf("failed to migrate existing state file to TPM-sealed format: %w", err)
}
}
return sf(logf, path)
}
}
if runtime.GOOS == "windows" {
path = TryWindowsAppDataMigration(logf, path)
}
if err := maybeMigrateLocalStateFile(logf, path); err != nil {
return nil, fmt.Errorf("failed to migrate existing TPM-sealed state file to plaintext format: %w", err)
}
return NewFileStore(logf, path)
}
@@ -84,6 +96,29 @@ func Register(prefix string, fn Provider) {
mak.Set(&knownStores, prefix, fn)
}
// RegisterForTest registers a prefix to be used for NewStore in tests. An
// existing registered prefix will be replaced.
func RegisterForTest(t testenv.TB, prefix string, fn Provider) {
if len(prefix) == 0 {
panic("prefix is empty")
}
old := maps.Clone(knownStores)
t.Cleanup(func() { knownStores = old })
mak.Set(&knownStores, prefix, fn)
}
// HasKnownProviderPrefix reports whether path uses one of the registered
// Provider prefixes.
func HasKnownProviderPrefix(path string) bool {
for prefix := range knownStores {
if strings.HasPrefix(path, prefix) {
return true
}
}
return false
}
// TryWindowsAppDataMigration attempts to copy the Windows state file
// from its old location to the new location. (Issue 2856)
//
@@ -186,3 +221,123 @@ func (s *FileStore) WriteState(id ipn.StateKey, bs []byte) error {
}
return atomicfile.WriteFile(s.path, bs, 0600)
}
func (s *FileStore) All() iter.Seq2[ipn.StateKey, []byte] {
return func(yield func(ipn.StateKey, []byte) bool) {
s.mu.Lock()
defer s.mu.Unlock()
for k, v := range s.cache {
if !yield(k, v) {
break
}
}
}
}
// Ensure FileStore implements ExportableStore for migration to/from
// tpm.tpmStore.
var _ ExportableStore = (*FileStore)(nil)
// ExportableStore is an ipn.StateStore that can export all of its contents.
// This interface is optional to implement, and used for migrating the state
// between different store implementations.
type ExportableStore interface {
ipn.StateStore
// All returns an iterator over all store keys. Using ReadState or
// WriteState is not safe while iterating and can lead to a deadlock. The
// order of keys in the iterator is not specified and may change between
// runs.
All() iter.Seq2[ipn.StateKey, []byte]
}
func maybeMigrateLocalStateFile(logf logger.Logf, path string) error {
path, toTPM := strings.CutPrefix(path, TPMPrefix)
// Extract JSON keys from the file on disk and guess what kind it is.
bs, err := os.ReadFile(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil
}
return err
}
var content map[string]any
if err := json.Unmarshal(bs, &content); err != nil {
return fmt.Errorf("failed to unmarshal %q: %w", path, err)
}
keys := slices.Sorted(maps.Keys(content))
tpmKeys := []string{"key", "nonce", "data"}
slices.Sort(tpmKeys)
// TPM-sealed files will have exactly these keys.
existingFileSealed := slices.Equal(keys, tpmKeys)
// Plaintext files for nodes that registered at least once will have this
// key, plus other dynamic ones.
_, existingFilePlaintext := content["_machinekey"]
isTPM := existingFileSealed && !existingFilePlaintext
if isTPM == toTPM {
// No migration needed.
return nil
}
newTPMStore, ok := knownStores[TPMPrefix]
if !ok {
return errors.New("this build does not support TPM integration")
}
// Open from (old format) and to (new format) stores for migration. The
// "to" store will be at tmpPath.
var from, to ipn.StateStore
tmpPath := path + ".tmp"
if toTPM {
// Migrate plaintext file to be TPM-sealed.
from, err = NewFileStore(logf, path)
if err != nil {
return fmt.Errorf("NewFileStore(%q): %w", path, err)
}
to, err = newTPMStore(logf, TPMPrefix+tmpPath)
if err != nil {
return fmt.Errorf("newTPMStore(%q): %w", tmpPath, err)
}
} else {
// Migrate TPM-selaed file to plaintext.
from, err = newTPMStore(logf, TPMPrefix+path)
if err != nil {
return fmt.Errorf("newTPMStore(%q): %w", path, err)
}
to, err = NewFileStore(logf, tmpPath)
if err != nil {
return fmt.Errorf("NewFileStore(%q): %w", tmpPath, err)
}
}
defer os.Remove(tmpPath)
fromExp, ok := from.(ExportableStore)
if !ok {
return fmt.Errorf("%T does not implement the exportableStore interface", from)
}
// Copy all the items. This is pretty inefficient, because both stores
// write the file to disk for each WriteState, but that's ok for a one-time
// migration.
for k, v := range fromExp.All() {
if err := to.WriteState(k, v); err != nil {
return err
}
}
// Finally, overwrite the state file with the new one we created at
// tmpPath.
if err := atomicfile.Rename(tmpPath, path); err != nil {
return err
}
if toTPM {
logf("migrated %q from plaintext to TPM-sealed format", path)
} else {
logf("migrated %q from TPM-sealed to plaintext format", path)
}
return nil
}