mirror of
https://github.com/rclone/rclone.git
synced 2024-12-02 21:37:31 +01:00
6427029c4e
* Update all dependencies * Remove all `[[constraint]]` from Gopkg.toml * Add in the minimum number of `[[override]]` to build * Remove go get of github.com/inconshreveable/mousetrap as it is vendored * Update docs with new policy on constraints
886 lines
30 KiB
Go
886 lines
30 KiB
Go
/*
|
|
Copyright 2015 Google Inc. All Rights Reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package bigtable
|
|
|
|
import (
|
|
"fmt"
|
|
"math"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"cloud.google.com/go/bigtable/internal/gax"
|
|
btopt "cloud.google.com/go/bigtable/internal/option"
|
|
"cloud.google.com/go/longrunning"
|
|
lroauto "cloud.google.com/go/longrunning/autogen"
|
|
"github.com/golang/protobuf/ptypes"
|
|
durpb "github.com/golang/protobuf/ptypes/duration"
|
|
"golang.org/x/net/context"
|
|
"google.golang.org/api/cloudresourcemanager/v1"
|
|
"google.golang.org/api/iterator"
|
|
"google.golang.org/api/option"
|
|
gtransport "google.golang.org/api/transport/grpc"
|
|
btapb "google.golang.org/genproto/googleapis/bigtable/admin/v2"
|
|
"google.golang.org/grpc"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/metadata"
|
|
"google.golang.org/grpc/status"
|
|
)
|
|
|
|
const adminAddr = "bigtableadmin.googleapis.com:443"
|
|
|
|
// AdminClient is a client type for performing admin operations within a specific instance.
|
|
type AdminClient struct {
|
|
conn *grpc.ClientConn
|
|
tClient btapb.BigtableTableAdminClient
|
|
lroClient *lroauto.OperationsClient
|
|
|
|
project, instance string
|
|
|
|
// Metadata to be sent with each request.
|
|
md metadata.MD
|
|
}
|
|
|
|
// NewAdminClient creates a new AdminClient for a given project and instance.
|
|
func NewAdminClient(ctx context.Context, project, instance string, opts ...option.ClientOption) (*AdminClient, error) {
|
|
o, err := btopt.DefaultClientOptions(adminAddr, AdminScope, clientUserAgent)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// Need to add scopes for long running operations (for create table & snapshots)
|
|
o = append(o, option.WithScopes(cloudresourcemanager.CloudPlatformScope))
|
|
o = append(o, opts...)
|
|
conn, err := gtransport.Dial(ctx, o...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("dialing: %v", err)
|
|
}
|
|
|
|
lroClient, err := lroauto.NewOperationsClient(ctx, option.WithGRPCConn(conn))
|
|
if err != nil {
|
|
// This error "should not happen", since we are just reusing old connection
|
|
// and never actually need to dial.
|
|
// If this does happen, we could leak conn. However, we cannot close conn:
|
|
// If the user invoked the function with option.WithGRPCConn,
|
|
// we would close a connection that's still in use.
|
|
// TODO(pongad): investigate error conditions.
|
|
return nil, err
|
|
}
|
|
|
|
return &AdminClient{
|
|
conn: conn,
|
|
tClient: btapb.NewBigtableTableAdminClient(conn),
|
|
lroClient: lroClient,
|
|
project: project,
|
|
instance: instance,
|
|
md: metadata.Pairs(resourcePrefixHeader, fmt.Sprintf("projects/%s/instances/%s", project, instance)),
|
|
}, nil
|
|
}
|
|
|
|
// Close closes the AdminClient.
|
|
func (ac *AdminClient) Close() error {
|
|
return ac.conn.Close()
|
|
}
|
|
|
|
func (ac *AdminClient) instancePrefix() string {
|
|
return fmt.Sprintf("projects/%s/instances/%s", ac.project, ac.instance)
|
|
}
|
|
|
|
// Tables returns a list of the tables in the instance.
|
|
func (ac *AdminClient) Tables(ctx context.Context) ([]string, error) {
|
|
ctx = mergeOutgoingMetadata(ctx, ac.md)
|
|
prefix := ac.instancePrefix()
|
|
req := &btapb.ListTablesRequest{
|
|
Parent: prefix,
|
|
}
|
|
|
|
var res *btapb.ListTablesResponse
|
|
err := gax.Invoke(ctx, func(ctx context.Context) error {
|
|
var err error
|
|
res, err = ac.tClient.ListTables(ctx, req)
|
|
return err
|
|
}, retryOptions...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
names := make([]string, 0, len(res.Tables))
|
|
for _, tbl := range res.Tables {
|
|
names = append(names, strings.TrimPrefix(tbl.Name, prefix+"/tables/"))
|
|
}
|
|
return names, nil
|
|
}
|
|
|
|
// TableConf contains all of the information necessary to create a table with column families.
|
|
type TableConf struct {
|
|
TableID string
|
|
SplitKeys []string
|
|
// Families is a map from family name to GCPolicy
|
|
Families map[string]GCPolicy
|
|
}
|
|
|
|
// CreateTable creates a new table in the instance.
|
|
// This method may return before the table's creation is complete.
|
|
func (ac *AdminClient) CreateTable(ctx context.Context, table string) error {
|
|
return ac.CreateTableFromConf(ctx, &TableConf{TableID: table})
|
|
}
|
|
|
|
// CreatePresplitTable creates a new table in the instance.
|
|
// The list of row keys will be used to initially split the table into multiple tablets.
|
|
// Given two split keys, "s1" and "s2", three tablets will be created,
|
|
// spanning the key ranges: [, s1), [s1, s2), [s2, ).
|
|
// This method may return before the table's creation is complete.
|
|
func (ac *AdminClient) CreatePresplitTable(ctx context.Context, table string, splitKeys []string) error {
|
|
return ac.CreateTableFromConf(ctx, &TableConf{TableID: table, SplitKeys: splitKeys})
|
|
}
|
|
|
|
// CreateTableFromConf creates a new table in the instance from the given configuration.
|
|
func (ac *AdminClient) CreateTableFromConf(ctx context.Context, conf *TableConf) error {
|
|
ctx = mergeOutgoingMetadata(ctx, ac.md)
|
|
var req_splits []*btapb.CreateTableRequest_Split
|
|
for _, split := range conf.SplitKeys {
|
|
req_splits = append(req_splits, &btapb.CreateTableRequest_Split{Key: []byte(split)})
|
|
}
|
|
var tbl btapb.Table
|
|
if conf.Families != nil {
|
|
tbl.ColumnFamilies = make(map[string]*btapb.ColumnFamily)
|
|
for fam, policy := range conf.Families {
|
|
tbl.ColumnFamilies[fam] = &btapb.ColumnFamily{GcRule: policy.proto()}
|
|
}
|
|
}
|
|
prefix := ac.instancePrefix()
|
|
req := &btapb.CreateTableRequest{
|
|
Parent: prefix,
|
|
TableId: conf.TableID,
|
|
Table: &tbl,
|
|
InitialSplits: req_splits,
|
|
}
|
|
_, err := ac.tClient.CreateTable(ctx, req)
|
|
return err
|
|
}
|
|
|
|
// CreateColumnFamily creates a new column family in a table.
|
|
func (ac *AdminClient) CreateColumnFamily(ctx context.Context, table, family string) error {
|
|
// TODO(dsymonds): Permit specifying gcexpr and any other family settings.
|
|
ctx = mergeOutgoingMetadata(ctx, ac.md)
|
|
prefix := ac.instancePrefix()
|
|
req := &btapb.ModifyColumnFamiliesRequest{
|
|
Name: prefix + "/tables/" + table,
|
|
Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
|
|
Id: family,
|
|
Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Create{Create: &btapb.ColumnFamily{}},
|
|
}},
|
|
}
|
|
_, err := ac.tClient.ModifyColumnFamilies(ctx, req)
|
|
return err
|
|
}
|
|
|
|
// DeleteTable deletes a table and all of its data.
|
|
func (ac *AdminClient) DeleteTable(ctx context.Context, table string) error {
|
|
ctx = mergeOutgoingMetadata(ctx, ac.md)
|
|
prefix := ac.instancePrefix()
|
|
req := &btapb.DeleteTableRequest{
|
|
Name: prefix + "/tables/" + table,
|
|
}
|
|
_, err := ac.tClient.DeleteTable(ctx, req)
|
|
return err
|
|
}
|
|
|
|
// DeleteColumnFamily deletes a column family in a table and all of its data.
|
|
func (ac *AdminClient) DeleteColumnFamily(ctx context.Context, table, family string) error {
|
|
ctx = mergeOutgoingMetadata(ctx, ac.md)
|
|
prefix := ac.instancePrefix()
|
|
req := &btapb.ModifyColumnFamiliesRequest{
|
|
Name: prefix + "/tables/" + table,
|
|
Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
|
|
Id: family,
|
|
Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Drop{Drop: true},
|
|
}},
|
|
}
|
|
_, err := ac.tClient.ModifyColumnFamilies(ctx, req)
|
|
return err
|
|
}
|
|
|
|
// TableInfo represents information about a table.
|
|
type TableInfo struct {
|
|
// DEPRECATED - This field is deprecated. Please use FamilyInfos instead.
|
|
Families []string
|
|
FamilyInfos []FamilyInfo
|
|
}
|
|
|
|
// FamilyInfo represents information about a column family.
|
|
type FamilyInfo struct {
|
|
Name string
|
|
GCPolicy string
|
|
}
|
|
|
|
// TableInfo retrieves information about a table.
|
|
func (ac *AdminClient) TableInfo(ctx context.Context, table string) (*TableInfo, error) {
|
|
ctx = mergeOutgoingMetadata(ctx, ac.md)
|
|
prefix := ac.instancePrefix()
|
|
req := &btapb.GetTableRequest{
|
|
Name: prefix + "/tables/" + table,
|
|
}
|
|
|
|
var res *btapb.Table
|
|
|
|
err := gax.Invoke(ctx, func(ctx context.Context) error {
|
|
var err error
|
|
res, err = ac.tClient.GetTable(ctx, req)
|
|
return err
|
|
}, retryOptions...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
ti := &TableInfo{}
|
|
for name, fam := range res.ColumnFamilies {
|
|
ti.Families = append(ti.Families, name)
|
|
ti.FamilyInfos = append(ti.FamilyInfos, FamilyInfo{Name: name, GCPolicy: GCRuleToString(fam.GcRule)})
|
|
}
|
|
return ti, nil
|
|
}
|
|
|
|
// SetGCPolicy specifies which cells in a column family should be garbage collected.
|
|
// GC executes opportunistically in the background; table reads may return data
|
|
// matching the GC policy.
|
|
func (ac *AdminClient) SetGCPolicy(ctx context.Context, table, family string, policy GCPolicy) error {
|
|
ctx = mergeOutgoingMetadata(ctx, ac.md)
|
|
prefix := ac.instancePrefix()
|
|
req := &btapb.ModifyColumnFamiliesRequest{
|
|
Name: prefix + "/tables/" + table,
|
|
Modifications: []*btapb.ModifyColumnFamiliesRequest_Modification{{
|
|
Id: family,
|
|
Mod: &btapb.ModifyColumnFamiliesRequest_Modification_Update{Update: &btapb.ColumnFamily{GcRule: policy.proto()}},
|
|
}},
|
|
}
|
|
_, err := ac.tClient.ModifyColumnFamilies(ctx, req)
|
|
return err
|
|
}
|
|
|
|
// DropRowRange permanently deletes a row range from the specified table.
|
|
func (ac *AdminClient) DropRowRange(ctx context.Context, table, rowKeyPrefix string) error {
|
|
ctx = mergeOutgoingMetadata(ctx, ac.md)
|
|
prefix := ac.instancePrefix()
|
|
req := &btapb.DropRowRangeRequest{
|
|
Name: prefix + "/tables/" + table,
|
|
Target: &btapb.DropRowRangeRequest_RowKeyPrefix{RowKeyPrefix: []byte(rowKeyPrefix)},
|
|
}
|
|
_, err := ac.tClient.DropRowRange(ctx, req)
|
|
return err
|
|
}
|
|
|
|
// CreateTableFromSnapshot creates a table from snapshot.
|
|
// The table will be created in the same cluster as the snapshot.
|
|
//
|
|
// This is a private alpha release of Cloud Bigtable snapshots. This feature
|
|
// is not currently available to most Cloud Bigtable customers. This feature
|
|
// might be changed in backward-incompatible ways and is not recommended for
|
|
// production use. It is not subject to any SLA or deprecation policy.
|
|
func (ac *AdminClient) CreateTableFromSnapshot(ctx context.Context, table, cluster, snapshot string) error {
|
|
ctx = mergeOutgoingMetadata(ctx, ac.md)
|
|
prefix := ac.instancePrefix()
|
|
snapshotPath := prefix + "/clusters/" + cluster + "/snapshots/" + snapshot
|
|
|
|
req := &btapb.CreateTableFromSnapshotRequest{
|
|
Parent: prefix,
|
|
TableId: table,
|
|
SourceSnapshot: snapshotPath,
|
|
}
|
|
op, err := ac.tClient.CreateTableFromSnapshot(ctx, req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resp := btapb.Table{}
|
|
return longrunning.InternalNewOperation(ac.lroClient, op).Wait(ctx, &resp)
|
|
}
|
|
|
|
const DefaultSnapshotDuration time.Duration = 0
|
|
|
|
// Creates a new snapshot in the specified cluster from the specified source table.
|
|
// Setting the ttl to `DefaultSnapshotDuration` will use the server side default for the duration.
|
|
//
|
|
// This is a private alpha release of Cloud Bigtable snapshots. This feature
|
|
// is not currently available to most Cloud Bigtable customers. This feature
|
|
// might be changed in backward-incompatible ways and is not recommended for
|
|
// production use. It is not subject to any SLA or deprecation policy.
|
|
func (ac *AdminClient) SnapshotTable(ctx context.Context, table, cluster, snapshot string, ttl time.Duration) error {
|
|
ctx = mergeOutgoingMetadata(ctx, ac.md)
|
|
prefix := ac.instancePrefix()
|
|
|
|
var ttlProto *durpb.Duration
|
|
|
|
if ttl > 0 {
|
|
ttlProto = ptypes.DurationProto(ttl)
|
|
}
|
|
|
|
req := &btapb.SnapshotTableRequest{
|
|
Name: prefix + "/tables/" + table,
|
|
Cluster: prefix + "/clusters/" + cluster,
|
|
SnapshotId: snapshot,
|
|
Ttl: ttlProto,
|
|
}
|
|
|
|
op, err := ac.tClient.SnapshotTable(ctx, req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resp := btapb.Snapshot{}
|
|
return longrunning.InternalNewOperation(ac.lroClient, op).Wait(ctx, &resp)
|
|
}
|
|
|
|
// Returns a SnapshotIterator for iterating over the snapshots in a cluster.
|
|
// To list snapshots across all of the clusters in the instance specify "-" as the cluster.
|
|
//
|
|
// This is a private alpha release of Cloud Bigtable snapshots. This feature
|
|
// is not currently available to most Cloud Bigtable customers. This feature
|
|
// might be changed in backward-incompatible ways and is not recommended for
|
|
// production use. It is not subject to any SLA or deprecation policy.
|
|
func (ac *AdminClient) ListSnapshots(ctx context.Context, cluster string) *SnapshotIterator {
|
|
ctx = mergeOutgoingMetadata(ctx, ac.md)
|
|
prefix := ac.instancePrefix()
|
|
clusterPath := prefix + "/clusters/" + cluster
|
|
|
|
it := &SnapshotIterator{}
|
|
req := &btapb.ListSnapshotsRequest{
|
|
Parent: clusterPath,
|
|
}
|
|
|
|
fetch := func(pageSize int, pageToken string) (string, error) {
|
|
req.PageToken = pageToken
|
|
if pageSize > math.MaxInt32 {
|
|
req.PageSize = math.MaxInt32
|
|
} else {
|
|
req.PageSize = int32(pageSize)
|
|
}
|
|
|
|
resp, err := ac.tClient.ListSnapshots(ctx, req)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
for _, s := range resp.Snapshots {
|
|
snapshotInfo, err := newSnapshotInfo(s)
|
|
if err != nil {
|
|
return "", fmt.Errorf("Failed to parse snapshot proto %v", err)
|
|
}
|
|
it.items = append(it.items, snapshotInfo)
|
|
}
|
|
return resp.NextPageToken, nil
|
|
}
|
|
bufLen := func() int { return len(it.items) }
|
|
takeBuf := func() interface{} { b := it.items; it.items = nil; return b }
|
|
|
|
it.pageInfo, it.nextFunc = iterator.NewPageInfo(fetch, bufLen, takeBuf)
|
|
|
|
return it
|
|
}
|
|
|
|
func newSnapshotInfo(snapshot *btapb.Snapshot) (*SnapshotInfo, error) {
|
|
nameParts := strings.Split(snapshot.Name, "/")
|
|
name := nameParts[len(nameParts)-1]
|
|
tablePathParts := strings.Split(snapshot.SourceTable.Name, "/")
|
|
tableId := tablePathParts[len(tablePathParts)-1]
|
|
|
|
createTime, err := ptypes.Timestamp(snapshot.CreateTime)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Invalid createTime: %v", err)
|
|
}
|
|
|
|
deleteTime, err := ptypes.Timestamp(snapshot.DeleteTime)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Invalid deleteTime: %v", err)
|
|
}
|
|
|
|
return &SnapshotInfo{
|
|
Name: name,
|
|
SourceTable: tableId,
|
|
DataSize: snapshot.DataSizeBytes,
|
|
CreateTime: createTime,
|
|
DeleteTime: deleteTime,
|
|
}, nil
|
|
}
|
|
|
|
// An EntryIterator iterates over log entries.
|
|
//
|
|
// This is a private alpha release of Cloud Bigtable snapshots. This feature
|
|
// is not currently available to most Cloud Bigtable customers. This feature
|
|
// might be changed in backward-incompatible ways and is not recommended for
|
|
// production use. It is not subject to any SLA or deprecation policy.
|
|
type SnapshotIterator struct {
|
|
items []*SnapshotInfo
|
|
pageInfo *iterator.PageInfo
|
|
nextFunc func() error
|
|
}
|
|
|
|
// PageInfo supports pagination. See https://godoc.org/google.golang.org/api/iterator package for details.
|
|
func (it *SnapshotIterator) PageInfo() *iterator.PageInfo {
|
|
return it.pageInfo
|
|
}
|
|
|
|
// Next returns the next result. Its second return value is iterator.Done
|
|
// (https://godoc.org/google.golang.org/api/iterator) if there are no more
|
|
// results. Once Next returns Done, all subsequent calls will return Done.
|
|
func (it *SnapshotIterator) Next() (*SnapshotInfo, error) {
|
|
if err := it.nextFunc(); err != nil {
|
|
return nil, err
|
|
}
|
|
item := it.items[0]
|
|
it.items = it.items[1:]
|
|
return item, nil
|
|
}
|
|
|
|
type SnapshotInfo struct {
|
|
Name string
|
|
SourceTable string
|
|
DataSize int64
|
|
CreateTime time.Time
|
|
DeleteTime time.Time
|
|
}
|
|
|
|
// Get snapshot metadata.
|
|
//
|
|
// This is a private alpha release of Cloud Bigtable snapshots. This feature
|
|
// is not currently available to most Cloud Bigtable customers. This feature
|
|
// might be changed in backward-incompatible ways and is not recommended for
|
|
// production use. It is not subject to any SLA or deprecation policy.
|
|
func (ac *AdminClient) SnapshotInfo(ctx context.Context, cluster, snapshot string) (*SnapshotInfo, error) {
|
|
ctx = mergeOutgoingMetadata(ctx, ac.md)
|
|
prefix := ac.instancePrefix()
|
|
clusterPath := prefix + "/clusters/" + cluster
|
|
snapshotPath := clusterPath + "/snapshots/" + snapshot
|
|
|
|
req := &btapb.GetSnapshotRequest{
|
|
Name: snapshotPath,
|
|
}
|
|
|
|
resp, err := ac.tClient.GetSnapshot(ctx, req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return newSnapshotInfo(resp)
|
|
}
|
|
|
|
// Delete a snapshot in a cluster.
|
|
//
|
|
// This is a private alpha release of Cloud Bigtable snapshots. This feature
|
|
// is not currently available to most Cloud Bigtable customers. This feature
|
|
// might be changed in backward-incompatible ways and is not recommended for
|
|
// production use. It is not subject to any SLA or deprecation policy.
|
|
func (ac *AdminClient) DeleteSnapshot(ctx context.Context, cluster, snapshot string) error {
|
|
ctx = mergeOutgoingMetadata(ctx, ac.md)
|
|
prefix := ac.instancePrefix()
|
|
clusterPath := prefix + "/clusters/" + cluster
|
|
snapshotPath := clusterPath + "/snapshots/" + snapshot
|
|
|
|
req := &btapb.DeleteSnapshotRequest{
|
|
Name: snapshotPath,
|
|
}
|
|
_, err := ac.tClient.DeleteSnapshot(ctx, req)
|
|
return err
|
|
}
|
|
|
|
// getConsistencyToken gets the consistency token for a table.
|
|
func (ac *AdminClient) getConsistencyToken(ctx context.Context, tableName string) (string, error) {
|
|
req := &btapb.GenerateConsistencyTokenRequest{
|
|
Name: tableName,
|
|
}
|
|
resp, err := ac.tClient.GenerateConsistencyToken(ctx, req)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return resp.GetConsistencyToken(), nil
|
|
}
|
|
|
|
// isConsistent checks if a token is consistent for a table.
|
|
func (ac *AdminClient) isConsistent(ctx context.Context, tableName, token string) (bool, error) {
|
|
req := &btapb.CheckConsistencyRequest{
|
|
Name: tableName,
|
|
ConsistencyToken: token,
|
|
}
|
|
var resp *btapb.CheckConsistencyResponse
|
|
|
|
// Retry calls on retryable errors to avoid losing the token gathered before.
|
|
err := gax.Invoke(ctx, func(ctx context.Context) error {
|
|
var err error
|
|
resp, err = ac.tClient.CheckConsistency(ctx, req)
|
|
return err
|
|
}, retryOptions...)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return resp.GetConsistent(), nil
|
|
}
|
|
|
|
// WaitForReplication waits until all the writes committed before the call started have been propagated to all the clusters in the instance via replication.
|
|
//
|
|
// This is a private alpha release of Cloud Bigtable replication. This feature
|
|
// is not currently available to most Cloud Bigtable customers. This feature
|
|
// might be changed in backward-incompatible ways and is not recommended for
|
|
// production use. It is not subject to any SLA or deprecation policy.
|
|
func (ac *AdminClient) WaitForReplication(ctx context.Context, table string) error {
|
|
// Get the token.
|
|
prefix := ac.instancePrefix()
|
|
tableName := prefix + "/tables/" + table
|
|
token, err := ac.getConsistencyToken(ctx, tableName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Periodically check if the token is consistent.
|
|
timer := time.NewTicker(time.Second * 10)
|
|
defer timer.Stop()
|
|
for {
|
|
consistent, err := ac.isConsistent(ctx, tableName, token)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if consistent {
|
|
return nil
|
|
}
|
|
// Sleep for a bit or until the ctx is cancelled.
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-timer.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
const instanceAdminAddr = "bigtableadmin.googleapis.com:443"
|
|
|
|
// InstanceAdminClient is a client type for performing admin operations on instances.
|
|
// These operations can be substantially more dangerous than those provided by AdminClient.
|
|
type InstanceAdminClient struct {
|
|
conn *grpc.ClientConn
|
|
iClient btapb.BigtableInstanceAdminClient
|
|
lroClient *lroauto.OperationsClient
|
|
|
|
project string
|
|
|
|
// Metadata to be sent with each request.
|
|
md metadata.MD
|
|
}
|
|
|
|
// NewInstanceAdminClient creates a new InstanceAdminClient for a given project.
|
|
func NewInstanceAdminClient(ctx context.Context, project string, opts ...option.ClientOption) (*InstanceAdminClient, error) {
|
|
o, err := btopt.DefaultClientOptions(instanceAdminAddr, InstanceAdminScope, clientUserAgent)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
o = append(o, opts...)
|
|
conn, err := gtransport.Dial(ctx, o...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("dialing: %v", err)
|
|
}
|
|
|
|
lroClient, err := lroauto.NewOperationsClient(ctx, option.WithGRPCConn(conn))
|
|
if err != nil {
|
|
// This error "should not happen", since we are just reusing old connection
|
|
// and never actually need to dial.
|
|
// If this does happen, we could leak conn. However, we cannot close conn:
|
|
// If the user invoked the function with option.WithGRPCConn,
|
|
// we would close a connection that's still in use.
|
|
// TODO(pongad): investigate error conditions.
|
|
return nil, err
|
|
}
|
|
|
|
return &InstanceAdminClient{
|
|
conn: conn,
|
|
iClient: btapb.NewBigtableInstanceAdminClient(conn),
|
|
lroClient: lroClient,
|
|
|
|
project: project,
|
|
md: metadata.Pairs(resourcePrefixHeader, "projects/"+project),
|
|
}, nil
|
|
}
|
|
|
|
// Close closes the InstanceAdminClient.
|
|
func (iac *InstanceAdminClient) Close() error {
|
|
return iac.conn.Close()
|
|
}
|
|
|
|
// StorageType is the type of storage used for all tables in an instance
|
|
type StorageType int
|
|
|
|
const (
|
|
SSD StorageType = iota
|
|
HDD
|
|
)
|
|
|
|
func (st StorageType) proto() btapb.StorageType {
|
|
if st == HDD {
|
|
return btapb.StorageType_HDD
|
|
}
|
|
return btapb.StorageType_SSD
|
|
}
|
|
|
|
// InstanceType is the type of the instance
|
|
type InstanceType int32
|
|
|
|
const (
|
|
PRODUCTION InstanceType = InstanceType(btapb.Instance_PRODUCTION)
|
|
DEVELOPMENT = InstanceType(btapb.Instance_DEVELOPMENT)
|
|
)
|
|
|
|
// InstanceInfo represents information about an instance
|
|
type InstanceInfo struct {
|
|
Name string // name of the instance
|
|
DisplayName string // display name for UIs
|
|
}
|
|
|
|
// InstanceConf contains the information necessary to create an Instance
|
|
type InstanceConf struct {
|
|
InstanceId, DisplayName, ClusterId, Zone string
|
|
// NumNodes must not be specified for DEVELOPMENT instance types
|
|
NumNodes int32
|
|
StorageType StorageType
|
|
InstanceType InstanceType
|
|
}
|
|
|
|
// InstanceWithClustersConfig contains the information necessary to create an Instance
|
|
type InstanceWithClustersConfig struct {
|
|
InstanceID, DisplayName string
|
|
Clusters []ClusterConfig
|
|
InstanceType InstanceType
|
|
}
|
|
|
|
var instanceNameRegexp = regexp.MustCompile(`^projects/([^/]+)/instances/([a-z][-a-z0-9]*)$`)
|
|
|
|
// CreateInstance creates a new instance in the project.
|
|
// This method will return when the instance has been created or when an error occurs.
|
|
func (iac *InstanceAdminClient) CreateInstance(ctx context.Context, conf *InstanceConf) error {
|
|
newConfig := InstanceWithClustersConfig{
|
|
InstanceID: conf.InstanceId,
|
|
DisplayName: conf.DisplayName,
|
|
InstanceType: conf.InstanceType,
|
|
Clusters: []ClusterConfig{
|
|
{
|
|
InstanceID: conf.InstanceId,
|
|
ClusterID: conf.ClusterId,
|
|
Zone: conf.Zone,
|
|
NumNodes: conf.NumNodes,
|
|
StorageType: conf.StorageType,
|
|
},
|
|
},
|
|
}
|
|
return iac.CreateInstanceWithClusters(ctx, &newConfig)
|
|
}
|
|
|
|
// CreateInstance creates a new instance with configured clusters in the project.
|
|
// This method will return when the instance has been created or when an error occurs.
|
|
//
|
|
// Instances with multiple clusters are part of a private alpha release of Cloud Bigtable replication.
|
|
// This feature is not currently available to most Cloud Bigtable customers. This feature
|
|
// might be changed in backward-incompatible ways and is not recommended for
|
|
// production use. It is not subject to any SLA or deprecation policy.
|
|
func (iac *InstanceAdminClient) CreateInstanceWithClusters(ctx context.Context, conf *InstanceWithClustersConfig) error {
|
|
ctx = mergeOutgoingMetadata(ctx, iac.md)
|
|
clusters := make(map[string]*btapb.Cluster)
|
|
for _, cluster := range conf.Clusters {
|
|
clusters[cluster.ClusterID] = cluster.proto(iac.project)
|
|
}
|
|
|
|
req := &btapb.CreateInstanceRequest{
|
|
Parent: "projects/" + iac.project,
|
|
InstanceId: conf.InstanceID,
|
|
Instance: &btapb.Instance{DisplayName: conf.DisplayName, Type: btapb.Instance_Type(conf.InstanceType)},
|
|
Clusters: clusters,
|
|
}
|
|
|
|
lro, err := iac.iClient.CreateInstance(ctx, req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resp := btapb.Instance{}
|
|
return longrunning.InternalNewOperation(iac.lroClient, lro).Wait(ctx, &resp)
|
|
}
|
|
|
|
// DeleteInstance deletes an instance from the project.
|
|
func (iac *InstanceAdminClient) DeleteInstance(ctx context.Context, instanceId string) error {
|
|
ctx = mergeOutgoingMetadata(ctx, iac.md)
|
|
req := &btapb.DeleteInstanceRequest{Name: "projects/" + iac.project + "/instances/" + instanceId}
|
|
_, err := iac.iClient.DeleteInstance(ctx, req)
|
|
return err
|
|
}
|
|
|
|
// Instances returns a list of instances in the project.
|
|
func (iac *InstanceAdminClient) Instances(ctx context.Context) ([]*InstanceInfo, error) {
|
|
ctx = mergeOutgoingMetadata(ctx, iac.md)
|
|
req := &btapb.ListInstancesRequest{
|
|
Parent: "projects/" + iac.project,
|
|
}
|
|
res, err := iac.iClient.ListInstances(ctx, req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(res.FailedLocations) > 0 {
|
|
// We don't have a good way to return a partial result in the face of some zones being unavailable.
|
|
// Fail the entire request.
|
|
return nil, status.Errorf(codes.Unavailable, "Failed locations: %v", res.FailedLocations)
|
|
}
|
|
|
|
var is []*InstanceInfo
|
|
for _, i := range res.Instances {
|
|
m := instanceNameRegexp.FindStringSubmatch(i.Name)
|
|
if m == nil {
|
|
return nil, fmt.Errorf("malformed instance name %q", i.Name)
|
|
}
|
|
is = append(is, &InstanceInfo{
|
|
Name: m[2],
|
|
DisplayName: i.DisplayName,
|
|
})
|
|
}
|
|
return is, nil
|
|
}
|
|
|
|
// InstanceInfo returns information about an instance.
|
|
func (iac *InstanceAdminClient) InstanceInfo(ctx context.Context, instanceId string) (*InstanceInfo, error) {
|
|
ctx = mergeOutgoingMetadata(ctx, iac.md)
|
|
req := &btapb.GetInstanceRequest{
|
|
Name: "projects/" + iac.project + "/instances/" + instanceId,
|
|
}
|
|
res, err := iac.iClient.GetInstance(ctx, req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
m := instanceNameRegexp.FindStringSubmatch(res.Name)
|
|
if m == nil {
|
|
return nil, fmt.Errorf("malformed instance name %q", res.Name)
|
|
}
|
|
return &InstanceInfo{
|
|
Name: m[2],
|
|
DisplayName: res.DisplayName,
|
|
}, nil
|
|
}
|
|
|
|
// ClusterConfig contains the information necessary to create a cluster
|
|
type ClusterConfig struct {
|
|
InstanceID, ClusterID, Zone string
|
|
NumNodes int32
|
|
StorageType StorageType
|
|
}
|
|
|
|
func (cc *ClusterConfig) proto(project string) *btapb.Cluster {
|
|
return &btapb.Cluster{
|
|
ServeNodes: cc.NumNodes,
|
|
DefaultStorageType: cc.StorageType.proto(),
|
|
Location: "projects/" + project + "/locations/" + cc.Zone,
|
|
}
|
|
}
|
|
|
|
// ClusterInfo represents information about a cluster.
|
|
type ClusterInfo struct {
|
|
Name string // name of the cluster
|
|
Zone string // GCP zone of the cluster (e.g. "us-central1-a")
|
|
ServeNodes int // number of allocated serve nodes
|
|
State string // state of the cluster
|
|
}
|
|
|
|
// CreateCluster creates a new cluster in an instance.
|
|
// This method will return when the cluster has been created or when an error occurs.
|
|
//
|
|
// This is a private alpha release of Cloud Bigtable replication. This feature
|
|
// is not currently available to most Cloud Bigtable customers. This feature
|
|
// might be changed in backward-incompatible ways and is not recommended for
|
|
// production use. It is not subject to any SLA or deprecation policy.
|
|
func (iac *InstanceAdminClient) CreateCluster(ctx context.Context, conf *ClusterConfig) error {
|
|
ctx = mergeOutgoingMetadata(ctx, iac.md)
|
|
|
|
req := &btapb.CreateClusterRequest{
|
|
Parent: "projects/" + iac.project + "/instances/" + conf.InstanceID,
|
|
ClusterId: conf.ClusterID,
|
|
Cluster: conf.proto(iac.project),
|
|
}
|
|
|
|
lro, err := iac.iClient.CreateCluster(ctx, req)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resp := btapb.Cluster{}
|
|
return longrunning.InternalNewOperation(iac.lroClient, lro).Wait(ctx, &resp)
|
|
}
|
|
|
|
// DeleteCluster deletes a cluster from an instance.
|
|
//
|
|
// This is a private alpha release of Cloud Bigtable replication. This feature
|
|
// is not currently available to most Cloud Bigtable customers. This feature
|
|
// might be changed in backward-incompatible ways and is not recommended for
|
|
// production use. It is not subject to any SLA or deprecation policy.
|
|
func (iac *InstanceAdminClient) DeleteCluster(ctx context.Context, instanceId, clusterId string) error {
|
|
ctx = mergeOutgoingMetadata(ctx, iac.md)
|
|
req := &btapb.DeleteClusterRequest{Name: "projects/" + iac.project + "/instances/" + instanceId + "/clusters/" + clusterId}
|
|
_, err := iac.iClient.DeleteCluster(ctx, req)
|
|
return err
|
|
}
|
|
|
|
// UpdateCluster updates attributes of a cluster
|
|
func (iac *InstanceAdminClient) UpdateCluster(ctx context.Context, instanceId, clusterId string, serveNodes int32) error {
|
|
ctx = mergeOutgoingMetadata(ctx, iac.md)
|
|
cluster := &btapb.Cluster{
|
|
Name: "projects/" + iac.project + "/instances/" + instanceId + "/clusters/" + clusterId,
|
|
ServeNodes: serveNodes}
|
|
lro, err := iac.iClient.UpdateCluster(ctx, cluster)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return longrunning.InternalNewOperation(iac.lroClient, lro).Wait(ctx, nil)
|
|
}
|
|
|
|
// Clusters lists the clusters in an instance.
|
|
func (iac *InstanceAdminClient) Clusters(ctx context.Context, instanceId string) ([]*ClusterInfo, error) {
|
|
ctx = mergeOutgoingMetadata(ctx, iac.md)
|
|
req := &btapb.ListClustersRequest{Parent: "projects/" + iac.project + "/instances/" + instanceId}
|
|
res, err := iac.iClient.ListClusters(ctx, req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// TODO(garyelliott): Deal with failed_locations.
|
|
var cis []*ClusterInfo
|
|
for _, c := range res.Clusters {
|
|
nameParts := strings.Split(c.Name, "/")
|
|
locParts := strings.Split(c.Location, "/")
|
|
cis = append(cis, &ClusterInfo{
|
|
Name: nameParts[len(nameParts)-1],
|
|
Zone: locParts[len(locParts)-1],
|
|
ServeNodes: int(c.ServeNodes),
|
|
State: c.State.String(),
|
|
})
|
|
}
|
|
return cis, nil
|
|
}
|
|
|
|
// GetCluster fetches a cluster in an instance
|
|
func (iac *InstanceAdminClient) GetCluster(ctx context.Context, instanceID, clusterID string) (*ClusterInfo, error) {
|
|
ctx = mergeOutgoingMetadata(ctx, iac.md)
|
|
req := &btapb.GetClusterRequest{Name: "projects/" + iac.project + "/instances/" + instanceID + "/clusters" + clusterID}
|
|
c, err := iac.iClient.GetCluster(ctx, req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
nameParts := strings.Split(c.Name, "/")
|
|
locParts := strings.Split(c.Location, "/")
|
|
cis := &ClusterInfo{
|
|
Name: nameParts[len(nameParts)-1],
|
|
Zone: locParts[len(locParts)-1],
|
|
ServeNodes: int(c.ServeNodes),
|
|
State: c.State.String(),
|
|
}
|
|
return cis, nil
|
|
}
|