Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • medina/public/cloud-evidence-collector
1 result
Show changes
Commits on Source (2)
Expand all Hide whitespace changes
Inline Side-by-side
cmd/.DS_Store 0 → 100644
View file @ bd82f92e
File added
cmd/collector/collector.go
View file @ bd82f92e
......@@ -31,36 +31,41 @@ import (
"collector"
"context"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"os"
"clouditor.io/clouditor/api/discovery"
"clouditor.io/clouditor/logging/formatter"
"clouditor.io/clouditor/rest"
"clouditor.io/clouditor/service"
"clouditor.io/clouditor/persistence"
"clouditor.io/clouditor/persistence/gorm"
server_clouditor "clouditor.io/clouditor/server"
"clouditor.io/clouditor/server/rest"
service_discovery "clouditor.io/clouditor/service/discovery"
"clouditor.io/clouditor/voc"
structpb "github.com/golang/protobuf/ptypes/struct"
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
grpc_auth "github.com/grpc-ecosystem/go-grpc-middleware/auth"
grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
grpc_ctxtags "github.com/grpc-ecosystem/go-grpc-middleware/tags"
"github.com/sirupsen/logrus"
"golang.org/x/oauth2/clientcredentials"
"google.golang.org/grpc"
"google.golang.org/grpc/reflection"
)
var (
log *logrus.Entry
server *grpc.Server
discoveryService discovery.DiscoveryServer
assessmentUrl string
jwksURL string
config clientcredentials.Config
storage persistence.Storage
assessmentUrl string
jwksURL string
config clientcredentials.Config
cloudServiceID string
resourceGroupId string
dbHost string
dbUser string
dbPassword string
// provider sets the cloud service provider an evidence collector instance (pod) should gather resources from
provider string
)
const (
......@@ -72,6 +77,12 @@ const (
OAuth2ClientSecret = "OAUTH2_CLIENT_SECRET"
OAuth2TokenURL = "OAUTH2_TOKEN_URL"
JWKSURL = "AUTH_JWKS_URL"
CloudServiceID = "CLOUD_SERVICE_ID"
ResourceGroupId = "RESOURCE_GROUP_ID"
PostgresHost = "POSTGRES_HOST"
PostgresUser = "POSTGRES_USER"
PostgresPassword = "POSTGRES_PASSWORD"
Provider = "PROVIDER"
)
func toStruct(r voc.IsCloudResource) (s *structpb.Value) {
......@@ -111,15 +122,33 @@ func init() {
// Get the JWKS URL of our auth server
jwksURL, _ = os.LookupEnv(JWKSURL)
cloudServiceID, ok = os.LookupEnv(CloudServiceID)
if !ok {
cloudServiceID = discovery.DefaultCloudServiceID
}
resourceGroupId, ok = os.LookupEnv(ResourceGroupId)
if !ok {
resourceGroupId = ""
}
// Get the OAuth credentials
config = clientcredentials.Config{}
config.ClientID, _ = os.LookupEnv(OAuth2ClientID)
config.ClientSecret, _ = os.LookupEnv(OAuth2ClientSecret)
config.TokenURL, _ = os.LookupEnv(OAuth2TokenURL)
// Get DB credentials
dbHost, _ = os.LookupEnv(PostgresHost)
dbUser, _ = os.LookupEnv(PostgresUser)
dbPassword, _ = os.LookupEnv(PostgresPassword)
}
func main() {
var textFormatter = logrus.TextFormatter{ForceColors: false, FullTimestamp: true}
var (
textFormatter = logrus.TextFormatter{ForceColors: false, FullTimestamp: true}
err error
)
log.Logger.Formatter = &textFormatter
fmt.Printf(`
......@@ -137,66 +166,72 @@ func main() {
log.Infof("Security Assessment URL is set to %s", assessmentUrl)
storage, err = gorm.NewStorage(gorm.WithPostgres(dbHost, 5432, dbUser, dbPassword, "postgres", ""))
if err != nil {
log.Errorf("Could not connect to storage (yet): %v", err)
os.Exit(1)
}
// Get provider (one is enough because 1 cloud service == 1 provider)
provider = os.Getenv(Provider)
if provider == "" {
log.Errorf("Exiting. No provider specified: %v", err)
os.Exit(1)
}
discoveryService = service_discovery.NewService(
service_discovery.WithOAuth2Authorizer(&config),
service_discovery.WithProviders([]string{service_discovery.ProviderAzure}),
service_discovery.WithProviders([]string{provider}),
service_discovery.WithAssessmentAddress(assessmentUrl),
service_discovery.WithCloudServiceID(cloudServiceID),
service_discovery.WithStorage(storage),
)
// Comment in if evidence collector should collect the evidences from the CSPs. Otherwise, example evidences are used.
//Start evidence collector
_, err := discoveryService.Start(context.Background(), &discovery.StartDiscoveryRequest{})
// Start evidence collector (possibly scoped to the resourcegroup)
if resourceGroupId == "" {
_, err = discoveryService.Start(context.Background(), &discovery.StartDiscoveryRequest{})
} else {
_, err = discoveryService.Start(context.Background(), &discovery.StartDiscoveryRequest{
ResourceGroup: &resourceGroupId,
})
}
if err != nil {
log.Errorf("could not collect evidences: %v", err)
}
grpcLogger := logrus.New()
grpcLogger.Formatter = &formatter.GRPCFormatter{TextFormatter: textFormatter}
grpcLoggerEntry := grpcLogger.WithField("component", "grpc")
log.Infof("Starting gRPC endpoint on :%d", grpcPort)
// create a new socket for gRPC communication
sock, err := net.Listen("tcp", fmt.Sprintf(":%d", grpcPort))
// Add grpc opts
grpcOpts := []grpc.ServerOption{
// Add max grpc message sizes
grpc.MaxRecvMsgSize(1024 * 1024 * 20),
grpc.MaxSendMsgSize(1024 * 1024 * 20)}
// Start the gRPC server
_, server, err = server_clouditor.StartGRPCServer(
fmt.Sprintf("0.0.0.0:%d", grpcPort),
server_clouditor.WithJWKS(jwksURL),
server_clouditor.WithDiscovery(discoveryService),
server_clouditor.WithReflection(),
server_clouditor.WithAdditionalGRPCOpts(grpcOpts),
)
if err != nil {
log.Errorf("could not listen: %v", err)
log.Errorf("Failed to serve gRPC endpoint: %s", err)
return
}
authConfig := service.ConfigureAuth(service.WithJWKSURL(jwksURL))
defer authConfig.Jwks.EndBackground()
server = grpc.NewServer(
grpc_middleware.WithUnaryServerChain(
grpc_ctxtags.UnaryServerInterceptor(grpc_ctxtags.WithFieldExtractor(grpc_ctxtags.CodeGenRequestFieldExtractor)),
grpc_logrus.UnaryServerInterceptor(grpcLoggerEntry),
grpc_auth.UnaryServerInterceptor(authConfig.AuthFunc),
),
grpc_middleware.WithStreamServerChain(
grpc_ctxtags.StreamServerInterceptor(grpc_ctxtags.WithFieldExtractor(grpc_ctxtags.CodeGenRequestFieldExtractor)),
grpc_logrus.StreamServerInterceptor(grpcLoggerEntry),
grpc_auth.StreamServerInterceptor(authConfig.AuthFunc),
))
discovery.RegisterDiscoveryServer(server, discoveryService)
// enable reflection, primary for testing in early stages
reflection.Register(server)
// start the gRPC-HTTP gateway
go func() {
err = rest.RunServer(context.Background(), grpcPort, httpPort)
if errors.Is(err, http.ErrServerClosed) {
os.Exit(0)
return
}
if err != nil {
log.Fatalf("failed to serve gRPC-HTTP gateway: %v", err)
}
}()
log.Infof("Starting gRPC endpoint on :%d", grpcPort)
// serve the gRPC socket
if err := server.Serve(sock); err != nil {
log.Infof("failed to serve gRPC endpoint: %s", err)
// Start the gRPC-HTTP gateway
err = rest.RunServer(context.Background(),
grpcPort,
httpPort,
)
if err != nil && err != http.ErrServerClosed {
log.Errorf("failed to serve gRPC-HTTP gateway: %v", err)
return
}
log.Infof("Stopping gRPC endpoint")
server.GracefulStop()
}
go.mod
View file @ bd82f92e
module collector
go 1.18
go 1.20
require (
clouditor.io/clouditor v1.4.15
github.com/golang/protobuf v1.5.2
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/sirupsen/logrus v1.8.1
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
google.golang.org/grpc v1.46.0
clouditor.io/clouditor v1.9.4-0.20230726134626-03c5b3f7af29
github.com/golang/protobuf v1.5.3
github.com/sirupsen/logrus v1.9.3
golang.org/x/oauth2 v0.8.0
google.golang.org/grpc v1.57.0-dev.0.20230612212144-642dd63a8527
)
require (
cloud.google.com/go v0.99.0 // indirect
github.com/Azure/azure-sdk-for-go v64.1.0+incompatible // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest v0.11.24 // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 // indirect
github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/MicahParks/keyfunc v1.1.0 // indirect
github.com/PuerkitoBio/purell v1.1.1 // indirect
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
github.com/aws/aws-sdk-go-v2 v1.16.3 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.0 // indirect
github.com/aws/aws-sdk-go-v2/config v1.15.0 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.10.0 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.10 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.7 // indirect
github.com/aws/aws-sdk-go-v2/service/ec2 v1.43.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.4 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.0 // indirect
github.com/aws/aws-sdk-go-v2/service/lambda v1.23.0 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.26.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.0 // indirect
github.com/aws/smithy-go v1.11.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0-beta.2 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/appservice/armappservice/v2 v2.1.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v3 v3.0.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dataprotection/armdataprotection v1.0.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork v1.1.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/security/armsecurity v0.11.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/sql/armsql v1.1.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.3.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/subscription/armsubscription v1.1.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
github.com/MicahParks/keyfunc/v2 v2.1.0 // indirect
github.com/aws/aws-sdk-go-v2 v1.19.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.24 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.23 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.27 // indirect
github.com/aws/aws-sdk-go-v2/service/ec2 v1.104.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.30 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.14.4 // indirect
github.com/aws/aws-sdk-go-v2/service/lambda v1.37.0 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.37.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/bmatcuk/doublestar/v4 v4.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/emicklei/go-restful v2.9.5+incompatible // indirect
github.com/go-co-op/gocron v1.13.0 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.5 // indirect
github.com/go-openapi/swag v0.19.14 // indirect
github.com/gofrs/uuid v4.1.0+incompatible // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/emicklei/go-restful/v3 v3.10.2 // indirect
github.com/envoyproxy/protoc-gen-validate v1.0.0 // indirect
github.com/fatih/structtag v1.2.0 // indirect
github.com/glebarez/go-sqlite v1.21.1 // indirect
github.com/glebarez/sqlite v1.8.0 // indirect
github.com/go-co-op/gocron v1.28.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
github.com/google/gnostic v0.6.8 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
github.com/golang/glog v1.1.1 // indirect
github.com/google/addlicense v1.1.0 // indirect
github.com/google/gnostic v0.6.9 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
github.com/iancoleman/strcase v0.2.0 // indirect
github.com/imdario/mergo v0.3.15 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgx/v5 v5.3.1 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/logrusorgru/aurora/v3 v3.0.0 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/lyft/protoc-gen-star v0.6.1 // indirect
github.com/lyft/protoc-gen-star/v2 v2.0.3 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-isatty v0.0.18 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/oxisto/oauth2go v0.5.12 // indirect
github.com/oxisto/oauth2go v0.9.0 // indirect
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/spf13/afero v1.9.5 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/srikrsna/protoc-gen-gotag v0.6.2 // indirect
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd // indirect
golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
go.uber.org/atomic v1.9.0 // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/net v0.11.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/term v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.9.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac // indirect
google.golang.org/protobuf v1.28.0 // indirect
google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
k8s.io/api v0.24.0 // indirect
k8s.io/apimachinery v0.24.0 // indirect
k8s.io/client-go v0.24.0 // indirect
k8s.io/klog/v2 v2.60.1 // indirect
k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
gorm.io/driver/postgres v1.5.0 // indirect
gorm.io/gorm v1.25.1 // indirect
k8s.io/api v0.27.1 // indirect
k8s.io/apimachinery v0.27.1 // indirect
k8s.io/client-go v0.27.1 // indirect
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect
modernc.org/libc v1.22.5 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.5.0 // indirect
modernc.org/sqlite v1.22.1 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
go.sum
View file @ bd82f92e
This diff is collapsed.