diff --git a/cmd/agent/container/setup.go b/cmd/agent/container/setup.go index 3060b00ad..de8756f0c 100644 --- a/cmd/agent/container/setup.go +++ b/cmd/agent/container/setup.go @@ -225,7 +225,7 @@ func fillContainerEnv(setupInfo *config.Result) error { } // merge config - newMergedConfig := &config.MergedDevContainerConfig{} + newMergedConfig := &config.MergedConfig{} err := config.SubstituteContainerEnv(config.ListToObject(os.Environ()), setupInfo.MergedConfig, newMergedConfig) if err != nil { return errors.Wrap(err, "substitute container env") diff --git a/cmd/agent/workspace/logs.go b/cmd/agent/workspace/logs.go index c714898c7..ab5321bd8 100644 --- a/cmd/agent/workspace/logs.go +++ b/cmd/agent/workspace/logs.go @@ -49,7 +49,7 @@ func (cmd *LogsCmd) Run(ctx context.Context) error { logger := log.Default.ErrorStreamOnly() // create new runner - runner, err := devcontainer.NewRunner(agent.ContainerDevPodHelperLocation, agent.DefaultAgentDownloadURL(), workspaceInfo, logger) + runner, err := devcontainer.NewRunner(agent.DevPodBinary, agent.DefaultAgentDownloadURL(), workspaceInfo, logger) if err != nil { return fmt.Errorf("create runner: %w", err) } diff --git a/cmd/agent/workspace/up.go b/cmd/agent/workspace/up.go index c81504d45..332a4547f 100644 --- a/cmd/agent/workspace/up.go +++ b/cmd/agent/workspace/up.go @@ -90,8 +90,7 @@ func (cmd *UpCmd) Run(ctx context.Context) error { } // start up - err = cmd.up(ctx, workspaceInfo, tunnelClient, logger) - if err != nil { + if err = cmd.up(ctx, workspaceInfo, tunnelClient, logger); err != nil { return errors.Wrap(err, "devcontainer up") } @@ -136,9 +135,10 @@ func (cmd *UpCmd) devPodUp(ctx context.Context, workspaceInfo *provider2.AgentWo } func CreateRunner(workspaceInfo *provider2.AgentWorkspaceInfo, log log.Logger) (devcontainer.Runner, error) { - return devcontainer.NewRunner(agent.ContainerDevPodHelperLocation, agent.DefaultAgentDownloadURL(), workspaceInfo, log) + return devcontainer.NewRunner(agent.DevPodBinary, agent.DefaultAgentDownloadURL(), workspaceInfo, log) } +// InitContentFolder creates the working directory for the devpod agent to store binaries / config func InitContentFolder(workspaceInfo *provider2.AgentWorkspaceInfo, log log.Logger) (bool, error) { // check if workspace content folder exists _, err := os.Stat(workspaceInfo.ContentFolder) @@ -182,6 +182,7 @@ func InitContentFolder(workspaceInfo *provider2.AgentWorkspaceInfo, log log.Logg return false, nil } +// initWorkspace sets up a fresh workspace so it can run the devcontainer via devpod, setting up credentials, container runtime etc. func initWorkspace(ctx context.Context, cancel context.CancelFunc, workspaceInfo *provider2.AgentWorkspaceInfo, debug, shouldInstallDaemon bool) (tunnel.TunnelClient, log.Logger, string, error) { // create a grpc client tunnelClient, err := tunnelserver.NewTunnelClient(os.Stdin, os.Stdout, true, 0) diff --git a/cmd/build.go b/cmd/build.go index e6eff8726..cb97a709b 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -25,10 +25,10 @@ type BuildCmd struct { *flags.GlobalFlags provider.CLIOptions - ProviderOptions []string + ProviderOptions []string // provider specific options overriding the context - SkipDelete bool - Machine string + SkipDelete bool // the command requires a workspace to run, setting true leaves this workspace running after building + Machine string // optional CLI arg to specify which machine to deploy devcontainer } // NewBuildCmd creates a new command @@ -64,7 +64,7 @@ func NewBuildCmd(flags *flags.GlobalFlags) *cobra.Command { // defer removal of temporary ssh config file defer os.Remove(sshConfigPath) - baseWorkspaceClient, err := workspace2.ResolveWorkspace( + baseWorkspaceClient, err := workspace2.Resolve( ctx, devPodConfig, "", @@ -101,7 +101,7 @@ func NewBuildCmd(flags *flags.GlobalFlags) *cobra.Command { return fmt.Errorf("building is currently not supported for proxy providers") } - return cmd.Run(ctx, workspaceClient) + return cmd.Build(ctx, workspaceClient, log.Default) }, } @@ -123,32 +123,17 @@ func NewBuildCmd(flags *flags.GlobalFlags) *cobra.Command { return buildCmd } -func (cmd *BuildCmd) Run(ctx context.Context, client client.WorkspaceClient) error { - // build workspace - err := cmd.build(ctx, client, log.Default) - if err != nil { - return err - } - - return nil -} - -func (cmd *BuildCmd) build(ctx context.Context, workspaceClient client.WorkspaceClient, log log.Logger) error { - err := workspaceClient.Lock(ctx) - if err != nil { +// Build uses a workspaceClient to execute a build pipeline remotely in the workspace +func (cmd *BuildCmd) Build(ctx context.Context, workspaceClient client.WorkspaceClient, log log.Logger) (err error) { + if err = workspaceClient.Lock(ctx); err != nil { return err } defer workspaceClient.Unlock() - err = startWait(ctx, workspaceClient, true, log) - if err != nil { + if err = startWait(ctx, workspaceClient, true, log); err != nil { return err } - return cmd.buildAgentClient(ctx, workspaceClient, log) -} - -func (cmd *BuildCmd) buildAgentClient(ctx context.Context, workspaceClient client.WorkspaceClient, log log.Logger) error { // compress info workspaceInfo, wInfo, err := workspaceClient.AgentInfo(cmd.CLIOptions) if err != nil { @@ -187,7 +172,7 @@ func (cmd *BuildCmd) buildAgentClient(ctx context.Context, workspaceClient clien writer := log.ErrorStreamOnly().Writer(logrus.InfoLevel, false) defer writer.Close() - errChan <- agent.InjectAgentAndExecute( + errChan <- agent.Inject( cancelCtx, func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { return workspaceClient.Command(ctx, client.CommandOptions{ diff --git a/cmd/delete.go b/cmd/delete.go index 7f5422f62..bed8bb122 100644 --- a/cmd/delete.go +++ b/cmd/delete.go @@ -53,7 +53,7 @@ func NewDeleteCmd(flags *flags.GlobalFlags) *cobra.Command { // Run runs the command logic func (cmd *DeleteCmd) Run(ctx context.Context, devPodConfig *config.Config, args []string) error { // try to load workspace - client, err := workspace2.GetWorkspace(devPodConfig, args, false, log.Default) + client, err := workspace2.Get(devPodConfig, args, false, log.Default) if err != nil { if len(args) == 0 { return fmt.Errorf("cannot delete workspace because there was an error loading the workspace: %w. Please specify the id of the workspace you want to delete. E.g. 'devpod delete my-workspace --force'", err) @@ -141,7 +141,7 @@ func (cmd *DeleteCmd) deleteSingleMachine(ctx context.Context, client client2.Ba } // try to find other workspace with same machine - workspaces, err := workspace2.ListWorkspaces(devPodConfig, log.Default) + workspaces, err := workspace2.List(devPodConfig, log.Default) if err != nil { return false, errors.Wrap(err, "list workspaces") } diff --git a/cmd/export.go b/cmd/export.go index 2ec4bf46b..054f04d9d 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -44,7 +44,7 @@ func NewExportCmd(flags *flags.GlobalFlags) *cobra.Command { func (cmd *ExportCmd) Run(ctx context.Context, devPodConfig *config.Config, args []string) error { // try to load workspace logger := log.Default.ErrorStreamOnly() - client, err := workspace2.GetWorkspace(devPodConfig, args, false, logger) + client, err := workspace2.Get(devPodConfig, args, false, logger) if err != nil { return err } diff --git a/cmd/helper/get_workspace_name.go b/cmd/helper/get_workspace_name.go index 61af896b3..da764cd9d 100644 --- a/cmd/helper/get_workspace_name.go +++ b/cmd/helper/get_workspace_name.go @@ -34,6 +34,6 @@ func (cmd *GetWorkspaceNameCommand) Run(ctx context.Context, args []string) erro return fmt.Errorf("workspace is missing") } - fmt.Print(workspace.GetWorkspaceName(args)) + fmt.Print(workspace.GetName(args)) return nil } diff --git a/cmd/helper/sh.go b/cmd/helper/sh.go index 7a49bb4a2..8439b8f1b 100644 --- a/cmd/helper/sh.go +++ b/cmd/helper/sh.go @@ -49,5 +49,5 @@ func (cmd *ShellCommand) Run(ctx context.Context, args []string) error { cmd.Command = string(content) } - return shell.ExecuteCommandWithShell(ctx, cmd.Command, os.Stdin, os.Stdout, os.Stderr, os.Environ()) + return shell.Execute(ctx, cmd.Command, os.Stdin, os.Stdout, os.Stderr, os.Environ()) } diff --git a/cmd/import.go b/cmd/import.go index 2487c0671..5750fc33e 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -261,7 +261,7 @@ func (cmd *ImportCmd) importProvider(devPodConfig *config.Config, exportConfig * } func (cmd *ImportCmd) checkForConflictingIDs(exportConfig *provider.ExportConfig, devPodConfig *config.Config, log log.Logger) error { - workspaces, err := workspace.ListWorkspaces(devPodConfig, log) + workspaces, err := workspace.List(devPodConfig, log) if err != nil { return fmt.Errorf("error listing workspaces: %w", err) } diff --git a/cmd/list.go b/cmd/list.go index eec6ffe87..57d36e795 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -53,7 +53,7 @@ func (cmd *ListCmd) Run(ctx context.Context) error { return err } - workspaces, err := workspace.ListWorkspaces(devPodConfig, log.Default) + workspaces, err := workspace.List(devPodConfig, log.Default) if err != nil { return err } diff --git a/cmd/logs.go b/cmd/logs.go index 871ea2c23..fff411a54 100644 --- a/cmd/logs.go +++ b/cmd/logs.go @@ -45,7 +45,7 @@ func (cmd *LogsCmd) Run(ctx context.Context, args []string) error { return err } - baseClient, err := workspace.GetWorkspace(devPodConfig, args, false, log.Default) + baseClient, err := workspace.Get(devPodConfig, args, false, log.Default) if err != nil { return err } @@ -82,7 +82,7 @@ func (cmd *LogsCmd) Run(ctx context.Context, args []string) error { stderr := log.ErrorStreamOnly().Writer(logrus.DebugLevel, false) defer stderr.Close() - errChan <- agent.InjectAgentAndExecute( + errChan <- agent.Inject( ctx, func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { return client.Command(ctx, clientpkg.CommandOptions{ diff --git a/cmd/logs_daemon.go b/cmd/logs_daemon.go index 0b943bc97..eddbb75fa 100644 --- a/cmd/logs_daemon.go +++ b/cmd/logs_daemon.go @@ -42,7 +42,7 @@ func (cmd *LogsDaemonCmd) Run(ctx context.Context, args []string) error { return err } - baseClient, err := workspace.GetWorkspace(devPodConfig, args, false, log.Default) + baseClient, err := workspace.Get(devPodConfig, args, false, log.Default) if err != nil { return err } else if baseClient.WorkspaceConfig().Machine.ID == "" { diff --git a/cmd/machine/delete.go b/cmd/machine/delete.go index 47fd8cf3d..a9b016ef0 100644 --- a/cmd/machine/delete.go +++ b/cmd/machine/delete.go @@ -51,7 +51,7 @@ func (cmd *DeleteCmd) Run(ctx context.Context, args []string) error { } // check if there are workspaces that still use this machine - workspaces, err := workspace.ListWorkspaces(devPodConfig, log.Default) + workspaces, err := workspace.List(devPodConfig, log.Default) if err != nil { return err } diff --git a/cmd/machine/ssh.go b/cmd/machine/ssh.go index 295fdaba6..03187c916 100644 --- a/cmd/machine/ssh.go +++ b/cmd/machine/ssh.go @@ -77,7 +77,7 @@ func (cmd *SSHCmd) Run(ctx context.Context, args []string) error { if cmd.Debug { command += " --debug" } - return devagent.InjectAgentAndExecute(ctx, func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + return devagent.Inject(ctx, func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { return machineClient.Command(ctx, client.CommandOptions{ Command: command, Stdin: stdin, diff --git a/cmd/provider/delete.go b/cmd/provider/delete.go index 285b0105d..32bdaf475 100644 --- a/cmd/provider/delete.go +++ b/cmd/provider/delete.go @@ -78,7 +78,7 @@ func (cmd *DeleteCmd) Run(ctx context.Context, args []string) error { func DeleteProvider(devPodConfig *config.Config, provider string, ignoreNotFound bool) error { // check if there are workspaces that still use this machine - workspaces, err := workspace.ListWorkspaces(devPodConfig, log.Default) + workspaces, err := workspace.List(devPodConfig, log.Default) if err != nil { return err } diff --git a/cmd/ssh.go b/cmd/ssh.go index 97ede95e8..b1c165057 100644 --- a/cmd/ssh.go +++ b/cmd/ssh.go @@ -75,7 +75,7 @@ func NewSSHCmd(flags *flags.GlobalFlags) *cobra.Command { return err } - client, err := workspace2.GetWorkspace(devPodConfig, args, true, log.Default.ErrorStreamOnly()) + client, err := workspace2.Get(devPodConfig, args, true, log.Default.ErrorStreamOnly()) if err != nil { return err } @@ -385,7 +385,7 @@ func (cmd *SSHCmd) startTunnel(ctx context.Context, devPodConfig *config.Config, } log.Debugf("Run outer container tunnel") - command := fmt.Sprintf("'%s' helper ssh-server --track-activity --stdio --workdir '%s'", agent.ContainerDevPodHelperLocation, workdir) + command := fmt.Sprintf("'%s' helper ssh-server --track-activity --stdio --workdir '%s'", agent.DevPodBinary, workdir) if cmd.Debug { command += " --debug" } @@ -424,7 +424,7 @@ func (cmd *SSHCmd) startServices( log log.Logger, ) { if cmd.User != "" { - err := tunnel.RunInContainer( + err := tunnel.StartCredentialsServer( ctx, devPodConfig, containerClient, @@ -474,7 +474,7 @@ func (cmd *SSHCmd) startProxyServices( writer := log.ErrorStreamOnly().Writer(logrus.DebugLevel, false) defer writer.Close() - command := fmt.Sprintf("'%s' agent container credentials-server --user '%s'", agent.ContainerDevPodHelperLocation, cmd.User) + command := fmt.Sprintf("'%s' agent container credentials-server --user '%s'", agent.DevPodBinary, cmd.User) if gitCredentials { command += " --configure-git-helper" } @@ -572,7 +572,7 @@ func (cmd *SSHCmd) setupGPGAgent( // Now we forward the agent socket to the remote, and setup remote gpg to use it // fix eventual permissions and so on forwardAgent := []string{ - agent.ContainerDevPodHelperLocation, + agent.DevPodBinary, } if log.GetLevel() == logrus.DebugLevel { diff --git a/cmd/status.go b/cmd/status.go index aed74b4fc..8454f2033 100644 --- a/cmd/status.go +++ b/cmd/status.go @@ -46,7 +46,7 @@ func NewStatusCmd(flags *flags.GlobalFlags) *cobra.Command { } logger := log.Default.ErrorStreamOnly() - client, err := workspace2.GetWorkspace(devPodConfig, args, false, logger) + client, err := workspace2.Get(devPodConfig, args, false, logger) if err != nil { return err } diff --git a/cmd/stop.go b/cmd/stop.go index e140bbd92..8ab63bf9c 100644 --- a/cmd/stop.go +++ b/cmd/stop.go @@ -34,7 +34,7 @@ func NewStopCmd(flags *flags.GlobalFlags) *cobra.Command { return err } - client, err := workspace2.GetWorkspace(devPodConfig, args, false, log.Default) + client, err := workspace2.Get(devPodConfig, args, false, log.Default) if err != nil { return err } @@ -88,7 +88,7 @@ func (cmd *StopCmd) stopSingleMachine(ctx context.Context, client client2.BaseWo } // try to find other workspace with same machine - workspaces, err := workspace2.ListWorkspaces(devPodConfig, log.Default) + workspaces, err := workspace2.List(devPodConfig, log.Default) if err != nil { return false, errors.Wrap(err, "list workspaces") } diff --git a/cmd/up.go b/cmd/up.go index 9efbc99af..e4d114535 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -48,9 +48,6 @@ import ( type UpCmd struct { provider2.CLIOptions *flags.GlobalFlags - - Machine string - ProviderOptions []string ConfigureSSH bool @@ -58,8 +55,8 @@ type UpCmd struct { OpenIDE bool SetupLoftPlatformAccess bool - SSHConfigPath string - + Machine string + SSHConfigPath string DotfilesSource string DotfilesScript string } @@ -72,14 +69,12 @@ func NewUpCmd(flags *flags.GlobalFlags) *cobra.Command { upCmd := &cobra.Command{ Use: "up", Short: "Starts a new workspace", - RunE: func(_ *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, args []string) (err error) { // try to parse flags from env - err := mergeDevPodUpOptions(&cmd.CLIOptions) - if err != nil { + if err := mergeDevPodUpOptions(&cmd.CLIOptions); err != nil { return err } - err = mergeEnvFromFiles(&cmd.CLIOptions) - if err != nil { + if err = mergeEnvFromFiles(&cmd.CLIOptions); err != nil { return err } @@ -107,7 +102,7 @@ func NewUpCmd(flags *flags.GlobalFlags) *cobra.Command { cmd.SSHConfigPath = devPodConfig.ContextOption(config.ContextOptionSSHConfigPath) } - client, err := workspace2.ResolveWorkspace( + client, err := workspace2.Resolve( ctx, devPodConfig, cmd.IDE, @@ -339,21 +334,20 @@ func (cmd *UpCmd) Run( return nil } +// devPodUp calls devPodUpProxy or devPodUpMachine based on the workspaceClient type func (cmd *UpCmd) devPodUp( ctx context.Context, devPodConfig *config.Config, client client2.BaseWorkspaceClient, log log.Logger, -) (*config2.Result, error) { - err := client.Lock(ctx) - if err != nil { +) (result *config2.Result, err error) { + + if err = client.Lock(ctx); err != nil { return nil, err } defer client.Unlock() // get result - var result *config2.Result - switch client := client.(type) { case client2.WorkspaceClient: result, err = cmd.devPodUpMachine(ctx, devPodConfig, client, log) @@ -378,6 +372,7 @@ func (cmd *UpCmd) devPodUp( return result, nil } +// devPodUpProxy runs "devpod agent workspace up" proxied by a local command, such as kubectl, based on the provider func (cmd *UpCmd) devPodUpProxy( ctx context.Context, client client2.ProxyClient, @@ -454,6 +449,7 @@ func (cmd *UpCmd) devPodUpProxy( return result, <-errChan } +// devPodUpMachine runs "devpod agent workspace up ..." on a remote machine to start the devcontainer func (cmd *UpCmd) devPodUpMachine( ctx context.Context, devPodConfig *config.Config, @@ -493,7 +489,7 @@ func (cmd *UpCmd) devPodUpMachine( } agentInjectFunc := func(cancelCtx context.Context, sshCmd string, sshTunnelStdinReader, sshTunnelStdoutWriter *os.File, writer io.WriteCloser) error { - return agent.InjectAgentAndExecute( + return agent.Inject( cancelCtx, func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { return client.Command(ctx, client2.CommandOptions{ @@ -560,8 +556,7 @@ func startJupyterNotebookInBrowser( logger log.Logger, ) error { if forwardGpg { - err := performGpgForwarding(client, logger) - if err != nil { + if err := performGpgForwarding(client, logger); err != nil { return err } } @@ -579,8 +574,7 @@ func startJupyterNotebookInBrowser( targetURL := fmt.Sprintf("http://localhost:%d/lab", jupyterPort) if jupyter.Options.GetValue(ideOptions, jupyter.OpenOption) == "true" { go func() { - err = open2.Open(ctx, targetURL, logger) - if err != nil { + if err = open2.Open(ctx, targetURL, logger); err != nil { logger.Errorf("error opening jupyter notebook: %v", err) } @@ -771,7 +765,7 @@ func startBrowserTunnel( } // run in container - err := tunnel.RunInContainer( + err := tunnel.StartCredentialsServer( ctx, devPodConfig, containerClient, @@ -797,6 +791,7 @@ func startBrowserTunnel( return nil } +// configureSSH updates the local SSH config to inject Host aliases and config needed for the tunnel func configureSSH(c *config.Config, client client2.BaseWorkspaceClient, sshConfigPath, user, workdir string, gpgagent bool) error { path, err := devssh.ResolveSSHConfigPath(sshConfigPath) if err != nil { @@ -820,6 +815,7 @@ func configureSSH(c *config.Config, client client2.BaseWorkspaceClient, sshConfi return nil } +// mergeDevPodUpOptions overrides the CLI arguments with environment variable options func mergeDevPodUpOptions(baseOptions *provider2.CLIOptions) error { oldOptions := *baseOptions found, err := clientimplementation.DecodeOptionsFromEnv( @@ -838,6 +834,7 @@ func mergeDevPodUpOptions(baseOptions *provider2.CLIOptions) error { return nil } +// mergeEnvFromFiles overrides the CLI arguments with devpod's config home files func mergeEnvFromFiles(baseOptions *provider2.CLIOptions) error { var variables []string for _, file := range baseOptions.WorkspaceEnvFile { @@ -880,6 +877,7 @@ func createSSHCommand( return exec.CommandContext(ctx, execPath, args...), nil } +// setupDotfiles download and source dot files from a repo or script on the remote workspace func setupDotfiles( dotfiles, script string, client client2.BaseWorkspaceClient, @@ -945,7 +943,7 @@ func setupDotfiles( client.Workspace(), "--log-output=raw", "--command", - agent.ContainerDevPodHelperLocation+" "+strings.Join(agentArguments, " "), + agent.DevPodBinary+" "+strings.Join(agentArguments, " "), ) if log.GetLevel() == logrus.DebugLevel { @@ -998,6 +996,7 @@ func setupGitSSHSignature(signingKey string, client client2.BaseWorkspaceClient, return nil } +// setupLoftPlatformAccess authenticates to loft platform if devpod pro is enabled func setupLoftPlatformAccess(context, provider, user string, client client2.BaseWorkspaceClient, log log.Logger) error { log.Infof("Setting up platform access") execPath, err := os.Executable() @@ -1010,7 +1009,7 @@ func setupLoftPlatformAccess(context, provider, user string, client client2.Base return fmt.Errorf("get port: %w", err) } - command := fmt.Sprintf("%v agent container setup-loft-platform-access --context %v --provider %v --port %v", agent.ContainerDevPodHelperLocation, context, provider, port) + command := fmt.Sprintf("%v agent container setup-loft-platform-access --context %v --provider %v --port %v", agent.DevPodBinary, context, provider, port) log.Debugf("Executing command -> %v", command) err = exec.Command( diff --git a/docs/pages/how-it-works/building-workspaces.mdx b/docs/pages/how-it-works/building-workspaces.mdx new file mode 100644 index 000000000..3226f55b1 --- /dev/null +++ b/docs/pages/how-it-works/building-workspaces.mdx @@ -0,0 +1,17 @@ +--- +title: How DevPod Builds Workspaces +sidebar_label: Building workspaces +--- + +Devpod provides the ability to build workspaces by taking a devcontainer.json and a git repository to compile an OCI compliant container with everything you need to develop +against using local tools. + +
+ DevPod Architecture +
DevPod - Container Diagram
+
+ +It does this by parsing the devcontainer.json, extracting the "features" and appending them as build stages to the base Dockerfile. The container is then built, depending on the driver +this could be docker, buildkit or kaniko and deployed with the configuration defined by your context. Optionally once the container is built, it can be pushed to a registry to cache for +other developers or in case you rebuild your workspace later. See #tutorials/reduce-build-times. + diff --git a/docs/pages/how-it-works/deploy-k8s.mdx b/docs/pages/how-it-works/deploy-k8s.mdx new file mode 100644 index 000000000..1ea2d3132 --- /dev/null +++ b/docs/pages/how-it-works/deploy-k8s.mdx @@ -0,0 +1,22 @@ +--- +title: How it works using kubernetes +sidebar_label: Deploying kubernetes +--- + +DevPod works the same with kubernetes as with Machines, the key difference is the secure tunnel is set up using the kubernetes control plane (e.g. kubectl ...) so an agent is not necessary +to be run on the kubernetes node. Instead devpod-provider-kubernetes simply wraps the appropriate `kubectl` commands to start and connect a workspace using a devcontainer. + +
+ DevPod Architecture +
DevPod - Container Diagram Kubernetes
+
+ +DevPod often has to build workspaces even when an "image" is specified in .devcontainer.json. This is because the devcontainer can contain "features" the cause the Dockerfile to be extended. +When this happens, or simply when "build" is used in .devcontainer.json, DevPod deploys an init container to the workspace that uses kaniko to first build your image (see #tutorials/reduce-build-times +for more details on kaniko) then executes the container's entrypoint in the pod's main container. While building, if REGISTRY_CACHE has been specified in the context options, kaniko will download +existing build layers from the registry to reduce the overall build time. + +
+ DevPod Architecture 2 +
DevPod - Component Diagram Kubernetes Build Process
+
\ No newline at end of file diff --git a/docs/pages/how-it-works/deploy-machines.mdx b/docs/pages/how-it-works/deploy-machines.mdx new file mode 100644 index 000000000..03d6bb453 --- /dev/null +++ b/docs/pages/how-it-works/deploy-machines.mdx @@ -0,0 +1,25 @@ +--- +title: How it works using VMs +sidebar_label: Deploying machines +--- + +Machines to DevPod are the infrastructure that ultimately run your devcontainer. Some providers such as gcloud, aws, digitalocean are "machine" providers since +they first setup a VM to deploy you container to. + +When DevPod starts a workspace, such as `devpod up`, devpod selects a provider based on your context and starts your devcontainer. If using a machine provider, devpod will check if it should create a VM first. +If so it uses your local environments credentials and the associated CLI tool, such as `aws` or `az` to create the infrastructure. Once started DevPod connects to the VM using the provider's specified tunnel, below +are some examples of providers and there secure tunnels. + + +- AWS: Instance connect +- GCloud: +- Azure: + +The dedvpod agent starts a SSH server using the STDIO of the secure tunnel in order for you local devpod CLI/UI to forward ports over the SSH connection. Once this is done devpod starts your local +IDE and connects it to the devcontainer via SSH. + +
+ DevPod Architecture +
DevPod - Component Diagram
+
+ diff --git a/docs/pages/how-it-works/deploying-workspaces.mdx b/docs/pages/how-it-works/deploying-workspaces.mdx new file mode 100644 index 000000000..3dffece40 --- /dev/null +++ b/docs/pages/how-it-works/deploying-workspaces.mdx @@ -0,0 +1,20 @@ +--- +title: How DevPod Deploys Workspaces +sidebar_label: Deploying workspaces +--- + +Devpod deploys workspaces using the "up" command, when executed devpod builds a devcontainer, if not already available, then uses the provider to deploy the devcontainer to a workspace. Below is a sequence diagram +of the main stages of the "up" command. + +
+ DevPod Up Sequence +
devpod up - Sequence Diagram
+
+ +First devpod checks if we need to create/start a machine to deploy the devcontainer to. Next we pull the source code and .devcontainer.json source from git or a local file and use this with the local environment +to build the workspace. Building is done by the agent since we need access to build tools such as buildkit or kaniko, i.e. `devpod workspace build`. The workspace now contains everything needed, +so devpod sets up a SSH connection to the devpod agent running alongside the container's control plane. + +The agent recieves "devpod agent workspace up" with the workspace spec serialised as workspace-info and uses the control plane (kube api server for k8s, docker daemon for anything else) to start the devcontainer. +Once started devpod deploys a daemon to monitor activity, optionally sets up any platform access for pro users then optionally retrieves credentials from the local environment before launching the IDE. Once the +IDE has started the deployment process has complete, devpod's agent daemon will continue to monitor the pod, if deployed, to put the machine or container to sleep when not in use. \ No newline at end of file diff --git a/docs/pages/how-it-works/overview.mdx b/docs/pages/how-it-works/overview.mdx new file mode 100644 index 000000000..cafb358ba --- /dev/null +++ b/docs/pages/how-it-works/overview.mdx @@ -0,0 +1,35 @@ +--- +title: How it works +sidebar_label: Overview +--- + +Devpod provides the ability to provision workspaces on any infrastructure. It does so by wrapping your conventential CLI tools such as kubectl, docker, gcloud etc to deploy your development environment +and set up everything required to run the dev container. While creating the workspace devpod deploys an agent to the machine running the container as well as to the container itself to provide useful +functions such as port forwarding, log streaming and more. Doing so it provides a control plane across your development environment. DevPod can be thought of as a minimal service mesh, specifically +designed to connect your local environment securely to your development container running in your cloud infrastructure. + +Devpod uses a client-agent architecture, where the client deploys it's own agent to host various servers, such as a grpc server or SSH server. +In this regard the system is not unlike a browser server architecture where the front end is deployed and executed on a remote host. There are several improvements this brings to our specific context: + - There can be no conflict of versions between client and server, since you install only one version of the client + - There is no infrastructure to manage for users + +To simplify debugging, devpod connects your local shell with the agent's STDIO so you can see what's happening locally and in the container at all times. + +All traffic between your local machine +and the workspace is tunneled through, ensuring secure communication using the cloud vendor's API. + +Below is a high level overview of how DevPod uses your local environment, a source repo and a devcontainer to deploy your workspace to the cloud. + +
+ DevPod Architecture +
DevPod - Component Diagram
+
+ +When you installed devpod a default "context" was created for you. This context is like a kubernetes context where devpod will use the config values you specify based on the context you are currently using. +This allows you to manage various workspaces across different providers. + +Devpod establishes a connection to the workspace using a vendor specific API. This vendor specific communication channel is referred to as the "tunnel". When you run a `devpod up` command, devpod selects a +provider based on your context and starts your devcontainer. If using a machine provider, devpod will check if it should create a VM first. Once the devcontainer +is running devpod deploys an agent to the container. The way in which devpod communicates with the workspace depends on the provider, this is known as the "tunnel". For AWS this is instance connect, kubernetes uses +the kubernetes control plane (kubectl), this connection is secured based on this tunnel. The devpod agent starts a SSH server using the STDIO of the secure tunnel in order for you local devpod CLI/UI to forward +ports over the SSH connection. Once this is done devpod starts your local IDE and connects it to the devcontainer via SSH. diff --git a/docs/pages/other-topics/advanced-guides/minikube-vscode-browser.mdx b/docs/pages/tutorials/minikube-vscode-browser.mdx similarity index 97% rename from docs/pages/other-topics/advanced-guides/minikube-vscode-browser.mdx rename to docs/pages/tutorials/minikube-vscode-browser.mdx index b4fb18bff..9cc7879cd 100644 --- a/docs/pages/other-topics/advanced-guides/minikube-vscode-browser.mdx +++ b/docs/pages/tutorials/minikube-vscode-browser.mdx @@ -3,9 +3,9 @@ title: Kubernetes Provider with Minikube and VS Code Browser sidebar_label: Kubernetes Provider with Minikube --- -import SetupVirtualbox from '../../fragments/setup-virtualbox.mdx' +import SetupVirtualbox from '../fragments/setup-virtualbox.mdx' -import VirtualboxUbuntu2204 from '../../fragments/virtualbox-ubuntu-22.04.mdx' +import VirtualboxUbuntu2204 from '../fragments/virtualbox-ubuntu-22.04.mdx' ## Purpose diff --git a/docs/pages/tutorials/reduce-build-times-with-cache.mdx b/docs/pages/tutorials/reduce-build-times-with-cache.mdx new file mode 100644 index 000000000..5383dc35b --- /dev/null +++ b/docs/pages/tutorials/reduce-build-times-with-cache.mdx @@ -0,0 +1,45 @@ +--- +title: Reduce build times with remote caching +sidebar_label: Reduce build times +--- + +Devpod provides the ability to use a registry as a remote backend cache for your container builds. To enable it perform + +``` +./devpod-cli context set-options -o REGISTRY_CACHE={registry} +``` + +where registry follows the syntax "\{domain\}/\{project\}/\{repo\}", such as gcr.io/my-project/my-dev-env + + +Ensure your docker environment has the image snapshotter enabled in your daemon (https://docs.docker.com/engine/storage/containerd/) +``` +cat /etc/docker/daemon.json + +{ + "features": { + "containerd-snapshotter": true + } +} +``` + +Then prebuild your image to populate the cache + +``` +devpod build my-workspace +``` + +Now the next time you or anyone in your team needs to build an image, it can pull from the cache. If you've made changes to your environment, then devpod will detect these changes are +build on top of the existing image layers. + +#### How does devpod detect changes? + +Devpod parses your Dockerfile and .devcontainer.json to detect file paths that can affect your build context. Devpod traverses these files and makes a hash of the file contents to use as the +container's image tag. When you make changes to these files, the hash will change causing a rebuild. Thanks to the remote cache the previous layers will be built so only changes are needed. To +reduce build times it is recommened to follow docker's best practices when writing Dockerfiles. + +#### Why Kaniko? + +When DevPod uses the kubernetes driver and needs to build a devcontainer, but your local environment does not have a container runtime, it builds the container in the cluster. It does so using +kaniko, kaniko builds the container in userspace and does not require super user priveledges. This is much more secure than docker in docker, where the docker daemon is either mounted locally on +the container or over a network within the cluster, neither being ideal. \ No newline at end of file diff --git a/docs/sidebars.js b/docs/sidebars.js index 6a086a846..4eb13967e 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -11,6 +11,32 @@ module.exports = { type: "doc", id: "what-is-devpod", }, + { + type: "category", + label: "How it works", + items: [ + { + type: "doc", + id: "how-it-works/overview", + }, + { + type: "doc", + id: "how-it-works/deploy-machines", + }, + { + type: "doc", + id: "how-it-works/deploy-k8s", + }, + { + type: "doc", + id: "how-it-works/building-workspaces", + }, + { + type: "doc", + id: "how-it-works/deploying-workspaces", + }, + ], + }, { type: "category", label: "Getting Started", @@ -135,6 +161,20 @@ module.exports = { }, ], }, + { + type: "category", + label: "Tutorials", + items: [ + { + type: "doc", + id: "tutorials/minikube-vscode-browser", + }, + { + type: "doc", + id: "tutorials/reduce-build-times-with-cache", + }, + ], + }, { type: "category", label: "Developing Providers", @@ -172,17 +212,7 @@ module.exports = { { type: "doc", id: "other-topics/mobile-support", - }, - { - type: "category", - label: "Advanced guides", - items: [ - { - type: "doc", - id: "other-topics/advanced-guides/minikube-vscode-browser", - }, - ], - }, + } ], }, { diff --git a/docs/static/media/arch_c4.png b/docs/static/media/arch_c4.png new file mode 100644 index 000000000..96f11b66e Binary files /dev/null and b/docs/static/media/arch_c4.png differ diff --git a/docs/static/media/arch_c4_machines.png b/docs/static/media/arch_c4_machines.png new file mode 100644 index 000000000..a362aae26 Binary files /dev/null and b/docs/static/media/arch_c4_machines.png differ diff --git a/docs/static/media/build_c4.png b/docs/static/media/build_c4.png new file mode 100644 index 000000000..542b47668 Binary files /dev/null and b/docs/static/media/build_c4.png differ diff --git a/docs/static/media/docker_build_c4.png b/docs/static/media/docker_build_c4.png new file mode 100644 index 000000000..ff4656f25 Binary files /dev/null and b/docs/static/media/docker_build_c4.png differ diff --git a/docs/static/media/up_sequence.png b/docs/static/media/up_sequence.png new file mode 100644 index 000000000..a9cada0eb Binary files /dev/null and b/docs/static/media/up_sequence.png differ diff --git a/docs/static/media/workspace_c4.png b/docs/static/media/workspace_c4.png new file mode 100644 index 000000000..7c1221592 Binary files /dev/null and b/docs/static/media/workspace_c4.png differ diff --git a/docs/uml/c4_build.puml b/docs/uml/c4_build.puml new file mode 100644 index 000000000..5771f3367 --- /dev/null +++ b/docs/uml/c4_build.puml @@ -0,0 +1,32 @@ +@startuml +!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml + +LAYOUT_TOP_DOWN() +HIDE_STEREOTYPE() + +title Component Diagram for DevPod Build Process + +Person(user, "DevPod", "CLI or UI") +Component(reg, "Registry", "Remote cache") +Component(ctx, "Context", "Build options, driver, registry cache etc.") +Component(cdf, "Compiled Dockerfile", "") + +Boundary(repo, "Git repo") { + Component(bctx, "Build context", "Source code, dot files etc.") + Component(df, "Dockerfile", "") +} + +Boundary(cluster, "Dev Container Spec") { + Component(dc, ".devcontainer.json", "Define dev env") + Component(fa, "Feature A", "docker in docker") + Component(fb, "Feature B", "git stuff") +} + +Rel(repo, user, " 1") +Rel(dc, user, " 2") +Rel(fa, user, " 3") +Rel(fb, user, " 3") +Rel_Right(ctx, user, " 4") +Rel(user, cdf, " 5") +Rel(cdf, reg, " 6") +@enduml \ No newline at end of file diff --git a/docs/uml/c4_component.puml b/docs/uml/c4_component.puml new file mode 100644 index 000000000..0e5f0e9ce --- /dev/null +++ b/docs/uml/c4_component.puml @@ -0,0 +1,36 @@ +@startuml +!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml + +HIDE_STEREOTYPE() +LAYOUT_TOP_DOWN() + +title Component diagram for DevPod with kubernetes + +Person(user, "DevPod", "CLI or UI") +System(prov, "Kubernetes Provider", "Devpod Provider") +Component(ide, "IDE", "Local IDE") +Component(ctx, "Context", "Config, working dir etc.") +Component(env, "Local Env", "Shell variables, git credentials etc.") + +System_Boundary(cluster, "Kubernetes Cluster") { + System(controlPlane, "kube API server", "Manages the cluster") + System_Boundary(ns, "devpod namespace") { + Container_Boundary(ws, "Workspace") { + Component(runner, "Kaniko runner", "") + Component(agent, "Agent", "") + Component(ssh, "SSH server", "") + ContainerDb(dir, "Workspace Directory", "Volume", "") + } + } +} + +BiRel_Right(user, prov, "devpod up ...") +BiRel(prov, controlPlane, " kubectl ...") +BiRel(controlPlane, agent, " ") +BiRel_Left(agent, ssh, "Tunnelled STDIO") +Rel(agent, runner, " ") +BiRel(ssh, ide, "SSH port forwarding") +Rel(user, ide, " ") +Rel_Up(user, ctx, " ") +Rel_Left(user, env, " ") +@enduml \ No newline at end of file diff --git a/docs/uml/c4_component_machine.puml b/docs/uml/c4_component_machine.puml new file mode 100644 index 000000000..d11890b98 --- /dev/null +++ b/docs/uml/c4_component_machine.puml @@ -0,0 +1,37 @@ +@startuml +!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml + +HIDE_STEREOTYPE() +LAYOUT_TOP_DOWN() + +title Component diagram for DevPod with machine providers + +Person(user, "DevPod", "CLI or UI") +System(prov, "DevPod Provider", "aws, gcloud etc.") +Component(ide, "IDE", "Local IDE") +Component(ctx, "Context", "Config, working dir etc.") +Component(env, "Local Env", "Shell variables, git credentials etc.") + +System_Boundary(cluster, "Virtual Machine") { + System(controlPlane, "DevPod Agent", "Manages the machine") + System(dockerd, "Docker daemon", "Manages the containers") + Container_Boundary(ws, "Workspace") { + Component(runner, "devcontainer", "") + Component(agent, "DevPod Container Agent", "") + Component(ssh, "SSH server", "") + ContainerDb(dir, "Workspace Directory", "Volume", "") + } +} + +BiRel(user, prov, " devpod up ...") +BiRel(prov, controlPlane, "secure provider specific tunnel") +Rel(controlPlane, dockerd, "docker run ...") +Rel(dockerd, ws, "") +Rel(agent, ssh, "") +Rel(agent, runner, "") +BiRel_Right(ide, ssh, "SSH port forwarding") +Rel(user, ide, "") +Rel(ctx, user, "") +Rel(env, user, "") +BiRel(controlPlane, agent, " SSH") +@enduml \ No newline at end of file diff --git a/docs/uml/c4_component_workspace.puml b/docs/uml/c4_component_workspace.puml new file mode 100644 index 000000000..9fb3de652 --- /dev/null +++ b/docs/uml/c4_component_workspace.puml @@ -0,0 +1,37 @@ +@startuml +!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml + +HIDE_STEREOTYPE() +LAYOUT_TOP_DOWN() + +title Component diagram for DevPod + +Person(user, "DevPod", "CLI or UI") +System(prov, "Provider", "Devpod Provider") +Component(ide, "IDE", "Local IDE") +Component(ctx, "Context", "Config, working dir etc.") +Component(env, "Local Env", "Shell variables, git credentials etc.") +Component(dc, ".devcontainer.json", "Dev container spec") +Component(git, "Git repo", "Project repo") + +System_Boundary(cluster, "Machine / Cluster") { + System(controlPlane, "Control plane", "Tunnels connection") + Container_Boundary(ws, "Workspace") { + Component(runner, "Dev container", "") + Component(agent, "Agent", "") + Component(ssh, "SSH server", "") + } +} + +BiRel(user, prov, "devpod up ...") +BiRel(prov, controlPlane, "") +BiRel(controlPlane, agent, " gRPC") +BiRel_Left(agent, ssh, "Tunnelled STDIO") +Rel(agent, runner, " ") +BiRel(ssh, ide, "SSH port forwarding") +Rel_Left(ide, user, " ") +Rel(ctx, user, " ") +Rel(env, user, " ") +Rel(git, user, " ") +Rel(dc, user,, " ") +@enduml \ No newline at end of file diff --git a/docs/uml/k8s_build.puml b/docs/uml/k8s_build.puml new file mode 100644 index 000000000..e50b3d351 --- /dev/null +++ b/docs/uml/k8s_build.puml @@ -0,0 +1,37 @@ +@startuml +!include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Container.puml +!include +!include + +HIDE_STEREOTYPE() +LAYOUT_TOP_DOWN() + +Person(user, "DevPod", "Uses kubectl") +System(prov, "Kubernetes Provider", "Devpod Provider") + +System_Boundary(host, "Kubernetes Cluster") { + System(controlPlane, "kube API server", "Manages the cluster") + + System_Boundary(devpodns, "devpod namespace") { + + Container_Boundary(kaniko1, "Workspace") { + ContainerDb(cache, "Build Cache", "Volume", "Local registry mirror") + ContainerDb(dir, "Workspace Directory", "Volume", "Build context") + Container(kr1, "Kaniko Runner", "kaniko:executor", "Runs container as linux process") + Container(kb1, "Kaniko Builder", "kaniko:executor", "Builds image in userspace") + } + } +} + +System_Boundary(registry, "Container Registry") { + System(registryCache, "Image Repository", "") + System(cacheRepo, "Cached Layer Repositories", "") +} + +Rel(user, prov, " devpod up ...") +Rel(prov, controlPlane, " kubectl ...") +Rel(controlPlane, kb1, " ") +Rel(user, registry, "devpod build ...") +Rel(registry, kb1, "") +Rel(kb1, kr1, " image") +@enduml \ No newline at end of file diff --git a/docs/uml/up_sequence.puml b/docs/uml/up_sequence.puml new file mode 100644 index 000000000..8861910b5 --- /dev/null +++ b/docs/uml/up_sequence.puml @@ -0,0 +1,67 @@ +@startuml +alt using machine provider +alt machine doesn't exist +DevPod -> Provider: devpod machine create +Provider --> DevPod: machine id +end +DevPod -> Provider: devpod machine start +Provider --> DevPod: machine id +end + +DevPod -> Repo: git pull +Repo --> DevPod: source code +alt devcontainer.json not in source repo +DevPod -> DevContainerSource: devpod crane pull +DevContainerSource --> DevPod: .devcontainer.json +end + +alt image not already available +DevPod -> Agent: devpod workspace build +activate Agent +Agent --> ContainerRuntime: build +ContainerRuntime --> Agent: container +Agent --> DevPod: +deactivate Agent +end + +DevPod -> Agent: devpod helper ssh-server --stdio +activate Agent +DevPod -> Agent: devpod agent daemon +DevPod -> Agent: devpod agent workspace up --workspace-info + +Agent -> ContainerRuntime: run +ContainerRuntime --> Agent: container id + +Agent -> ContainerAgent: devpod helper ssh-server --stdio +activate ContainerAgent + +Agent -> ContainerAgent: devpod agent container setup +Agent -> ContainerAgent: devpod agent container credentials-server +alt if EXIT_AFTER_TIMEOUT +Agent -> ContainerAgent: devpod agent container daemon +end + +alt if pro user +Agent -> ContainerAgent: devpod agent container setup-loft-platform-access +ContainerAgent -> Platform: Setup pro +Platform --> ContainerAgent: platform creds +end + +alt if GIT_SSH_SIGNATURE_FORWARDING +Agent -> DevPod: devpod agent git-credentials +DevPod --> Agent: git creds +end +alt if SSH_INJECT_DOCKER_CREDENTIALS +Agent -> DevPod: devpod agent docker-credentials +DevPod --> Agent: docker creds +end + +Agent -> ContainerAgent: Run life cycle hooks + +ContainerAgent --> Agent: +deactivate ContainerAgent +Agent --> DevPod: +deactivate Agent + +DevPod -> IDE: Start +@enduml \ No newline at end of file diff --git a/e2e/tests/build/build.go b/e2e/tests/build/build.go index c12e4e0c1..a1cd49113 100644 --- a/e2e/tests/build/build.go +++ b/e2e/tests/build/build.go @@ -281,15 +281,15 @@ var _ = DevPodDescribe("devpod build test suite", func() { }) }) -func getDevcontainerConfig(dir string) *config.DevContainerConfig { - return &config.DevContainerConfig{ - DevContainerConfigBase: config.DevContainerConfigBase{ +func getDevcontainerConfig(dir string) *config.Config { + return &config.Config{ + ConfigBase: config.ConfigBase{ Name: "Build Example", }, - DevContainerActions: config.DevContainerActions{}, - NonComposeBase: config.NonComposeBase{}, - ImageContainer: config.ImageContainer{}, - ComposeContainer: config.ComposeContainer{}, + Actions: config.Actions{}, + NonComposeBase: config.NonComposeBase{}, + ImageContainer: config.ImageContainer{}, + ComposeContainer: config.ComposeContainer{}, DockerfileContainer: config.DockerfileContainer{ Build: &config.ConfigBuildOptions{ Dockerfile: "Dockerfile", diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 9bec10b8e..219271bc1 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -24,7 +24,7 @@ import ( const DefaultInactivityTimeout = time.Minute * 20 -const ContainerDevPodHelperLocation = "/usr/local/bin/devpod" +const DevPodBinary = "/usr/local/bin/devpod" const RemoteDevPodHelperLocation = "/tmp/devpod" @@ -316,15 +316,16 @@ func Tunnel( timeout time.Duration, ) error { // inject agent - err := InjectAgent(ctx, func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + execFn := func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { return exec(ctx, "root", command, stdin, stdout, stderr) - }, false, ContainerDevPodHelperLocation, DefaultAgentDownloadURL(), false, log, timeout) + } + err := Inject(ctx, execFn, false, DevPodBinary, DefaultAgentDownloadURL(), false, "", nil, nil, nil, log, timeout) if err != nil { return err } // build command - command := fmt.Sprintf("'%s' helper ssh-server --stdio", ContainerDevPodHelperLocation) + command := fmt.Sprintf("'%s' helper ssh-server --stdio", DevPodBinary) if log.GetLevel() == logrus.DebugLevel { command += " --debug" } diff --git a/pkg/agent/inject.go b/pkg/agent/inject.go index 24322c8de..893045470 100644 --- a/pkg/agent/inject.go +++ b/pkg/agent/inject.go @@ -18,35 +18,9 @@ import ( "github.com/pkg/errors" ) -var waitForInstanceConnectionTimeout = time.Minute * 5 +var connectionTimeout = time.Minute * 5 -func InjectAgent( - ctx context.Context, - exec inject.ExecFunc, - local bool, - remoteAgentPath, - downloadURL string, - preferDownload bool, - log log.Logger, - timeout time.Duration, -) error { - return InjectAgentAndExecute( - ctx, - exec, - local, - remoteAgentPath, - downloadURL, - preferDownload, - "", - nil, - nil, - nil, - log, - timeout, - ) -} - -func InjectAgentAndExecute( +func Inject( ctx context.Context, exec inject.ExecFunc, local bool, @@ -67,7 +41,7 @@ func InjectAgentAndExecute( } log.Debugf("Execute command locally") - return shell.ExecuteCommandWithShell(ctx, command, stdin, stdout, stderr, nil) + return shell.Execute(ctx, command, stdin, stdout, stderr, nil) } defer log.Debugf("Done InjectAgentAndExecute") @@ -122,7 +96,7 @@ func InjectAgentAndExecute( log, ) if err != nil { - if time.Since(now) > waitForInstanceConnectionTimeout { + if time.Since(now) > connectionTimeout { return errors.Wrap(err, "timeout waiting for instance connection") } else if wasExecuted { return errors.Wrapf(err, "agent error: %s", buf.String()) diff --git a/pkg/client/clientimplementation/workspace_client.go b/pkg/client/clientimplementation/workspace_client.go index 38bd7fcd5..c4a4f41bd 100644 --- a/pkg/client/clientimplementation/workspace_client.go +++ b/pkg/client/clientimplementation/workspace_client.go @@ -589,7 +589,7 @@ func RunCommand(ctx context.Context, command types.StrArray, environ []string, s // use shell if command length is equal 1 if len(command) == 1 { - return shell.ExecuteCommandWithShell(ctx, command[0], stdin, stdout, stderr, environ) + return shell.Execute(ctx, command[0], stdin, stdout, stderr, environ) } // run command diff --git a/pkg/devcontainer/build.go b/pkg/devcontainer/build.go index 4c01bb484..4d68f5170 100644 --- a/pkg/devcontainer/build.go +++ b/pkg/devcontainer/build.go @@ -187,7 +187,7 @@ func (r *runner) buildAndExtendImage( return r.buildImage(ctx, parsedConfig, substitutionContext, imageBuildInfo, extendedBuildInfo, dockerFilePath, string(dockerFileContent), options) } -func (r *runner) getDockerfilePath(parsedConfig *config.DevContainerConfig) (string, error) { +func (r *runner) getDockerfilePath(parsedConfig *config.Config) (string, error) { if parsedConfig.Origin == "" { return "", fmt.Errorf("couldn't find path where config was loaded from") } diff --git a/pkg/devcontainer/compose.go b/pkg/devcontainer/compose.go index 15eea6131..19c260ef2 100644 --- a/pkg/devcontainer/compose.go +++ b/pkg/devcontainer/compose.go @@ -603,7 +603,7 @@ func (r *runner) extendedDockerComposeBuild(composeService *composetypes.Service func (r *runner) extendedDockerComposeUp( parsedConfig *config.SubstitutedConfig, - mergedConfig *config.MergedDevContainerConfig, + mergedConfig *config.MergedConfig, composeHelper *compose.ComposeHelper, composeService *composetypes.ServiceConfig, originalImageName, @@ -640,7 +640,7 @@ func (r *runner) extendedDockerComposeUp( func (r *runner) generateDockerComposeUpProject( parsedConfig *config.SubstitutedConfig, - mergedConfig *config.MergedDevContainerConfig, + mergedConfig *config.MergedConfig, composeHelper *compose.ComposeHelper, composeService *composetypes.ServiceConfig, originalImageName, @@ -804,6 +804,6 @@ func mappingToMap(mapping composetypes.MappingWithEquals) map[string]string { return ret } -func isDockerComposeConfig(config *config.DevContainerConfig) bool { +func isDockerComposeConfig(config *config.Config) bool { return len(config.DockerComposeFile) > 0 } diff --git a/pkg/devcontainer/config.go b/pkg/devcontainer/config.go index 3a10e10ca..1012965c4 100644 --- a/pkg/devcontainer/config.go +++ b/pkg/devcontainer/config.go @@ -12,9 +12,9 @@ import ( "github.com/pkg/errors" ) -func (r *runner) getRawConfig(options provider2.CLIOptions) (*config.DevContainerConfig, error) { +func (r *runner) getRawConfig(options provider2.CLIOptions) (*config.Config, error) { if r.WorkspaceConfig.Workspace.DevContainerConfig != nil { - rawParsedConfig := config.CloneDevContainerConfig(r.WorkspaceConfig.Workspace.DevContainerConfig) + rawParsedConfig := config.CloneConfig(r.WorkspaceConfig.Workspace.DevContainerConfig) if r.WorkspaceConfig.Workspace.DevContainerPath != "" { rawParsedConfig.Origin = path.Join(filepath.ToSlash(r.LocalWorkspaceFolder), r.WorkspaceConfig.Workspace.DevContainerPath) } else { @@ -22,8 +22,8 @@ func (r *runner) getRawConfig(options provider2.CLIOptions) (*config.DevContaine } return rawParsedConfig, nil } else if r.WorkspaceConfig.Workspace.Source.Container != "" { - return &config.DevContainerConfig{ - DevContainerConfigBase: config.DevContainerConfigBase{ + return &config.Config{ + ConfigBase: config.ConfigBase{ // Default workspace directory for containers // Upon inspecting the container, this would be updated to the correct folder, if found set WorkspaceFolder: "/", @@ -39,7 +39,7 @@ func (r *runner) getRawConfig(options provider2.CLIOptions) (*config.DevContaine return nil, err } - return config.ParseDevContainerJSON( + return config.Parse( localWorkspaceFolder, r.WorkspaceConfig.Workspace.DevContainerPath, ) @@ -53,7 +53,7 @@ func (r *runner) getRawConfig(options provider2.CLIOptions) (*config.DevContaine } // parse the devcontainer json - rawParsedConfig, err := config.ParseDevContainerJSON( + rawParsedConfig, err := config.Parse( localWorkspaceFolder, r.WorkspaceConfig.Workspace.DevContainerPath, ) @@ -69,8 +69,8 @@ func (r *runner) getRawConfig(options provider2.CLIOptions) (*config.DevContaine return rawParsedConfig, nil } -func (r *runner) getDefaultConfig(options provider2.CLIOptions) (*config.DevContainerConfig, error) { - defaultConfig := &config.DevContainerConfig{} +func (r *runner) getDefaultConfig(options provider2.CLIOptions) (*config.Config, error) { + defaultConfig := &config.Config{} if options.FallbackImage != "" { r.Log.Infof("Using fallback image %s", options.FallbackImage) defaultConfig.ImageContainer = config.ImageContainer{ @@ -82,7 +82,7 @@ func (r *runner) getDefaultConfig(options provider2.CLIOptions) (*config.DevCont } defaultConfig.Origin = path.Join(filepath.ToSlash(r.LocalWorkspaceFolder), ".devcontainer.json") - err := config.SaveDevContainerJSON(defaultConfig) + err := config.Save(defaultConfig) if err != nil { return nil, errors.Wrap(err, "write default devcontainer.json") } @@ -100,7 +100,7 @@ func (r *runner) getSubstitutedConfig(options provider2.CLIOptions) (*config.Sub func (r *runner) substitute( options provider2.CLIOptions, - rawParsedConfig *config.DevContainerConfig, + rawParsedConfig *config.Config, ) (*config.SubstitutedConfig, *config.SubstitutionContext, error) { configFile := rawParsedConfig.Origin @@ -120,7 +120,7 @@ func (r *runner) substitute( } // substitute & load - parsedConfig := &config.DevContainerConfig{} + parsedConfig := &config.Config{} err := config.Substitute(substitutionContext, rawParsedConfig, parsedConfig) if err != nil { return nil, nil, err diff --git a/pkg/devcontainer/config/build.go b/pkg/devcontainer/config/build.go index 69e59fdbf..5fdc7bad5 100644 --- a/pkg/devcontainer/config/build.go +++ b/pkg/devcontainer/config/build.go @@ -16,6 +16,7 @@ func GetDockerLabelForID(id string) []string { return []string{DockerIDLabel + "=" + id} } +// BuildInfo is returned once an image has been built and contains details about the image type BuildInfo struct { ImageDetails *ImageDetails ImageMetadata *ImageMetadataConfig @@ -25,6 +26,7 @@ type BuildInfo struct { Dockerless *BuildInfoDockerless } +// BuildInfoDockerless is the specific config used by kaniko to build a "dockerless image" type BuildInfoDockerless struct { Context string Dockerfile string @@ -35,6 +37,7 @@ type BuildInfoDockerless struct { User string } +// ImageBuildInfo contains the spec to use when building an image type ImageBuildInfo struct { User string Metadata *ImageMetadataConfig diff --git a/pkg/devcontainer/config/config.go b/pkg/devcontainer/config/config.go index ec3759d0b..fda4e1a34 100644 --- a/pkg/devcontainer/config/config.go +++ b/pkg/devcontainer/config/config.go @@ -10,8 +10,8 @@ import ( "github.com/loft-sh/devpod/pkg/types" ) -type MergedDevContainerConfig struct { - DevContainerConfigBase `json:",inline"` +type MergedConfig struct { + ConfigBase `json:",inline"` UpdatedConfigProperties `json:",inline"` NonComposeBase `json:",inline"` ImageContainer `json:",inline"` @@ -23,27 +23,27 @@ type MergedDevContainerConfig struct { Origin string `json:"-"` } -type DevContainerConfig struct { - DevContainerConfigBase `json:",inline"` - DevContainerActions `json:",inline"` - NonComposeBase `json:",inline"` - ImageContainer `json:",inline"` - ComposeContainer `json:",inline"` - DockerfileContainer `json:",inline"` - RunningContainer `json:",inline"` +type Config struct { + ConfigBase `json:",inline"` + Actions `json:",inline"` + NonComposeBase `json:",inline"` + ImageContainer `json:",inline"` + ComposeContainer `json:",inline"` + DockerfileContainer `json:",inline"` + RunningContainer `json:",inline"` // Origin is the origin from where this config was loaded Origin string `json:"-"` } -func CloneDevContainerConfig(config *DevContainerConfig) *DevContainerConfig { - out := &DevContainerConfig{} +func CloneConfig(config *Config) *Config { + out := &Config{} _ = Convert(config, out) out.Origin = config.Origin return out } -type DevContainerConfigBase struct { +type ConfigBase struct { // A name for the dev container which can be displayed to the user. Name string `json:"name,omitempty"` @@ -105,7 +105,7 @@ type DevContainerConfigBase struct { DevPort int `json:"devPort,omitempty"` } -type DevContainerActions struct { +type Actions struct { // A command to run when creating the container. This command is run after "initializeCommand" and before "updateContentCommand". If this is a single string, it will be run in a shell. If this is an array of strings, it will be run as a single command without shell. OnCreateCommand types.LifecycleHook `json:"onCreateCommand,omitempty"` @@ -361,7 +361,7 @@ func (m *Mount) String() string { return strings.Join(components, ",") } -func GetContextPath(parsedConfig *DevContainerConfig) string { +func GetContextPath(parsedConfig *Config) string { context := parsedConfig.GetContext() dockerfilePath := parsedConfig.GetDockerfile() diff --git a/pkg/devcontainer/config/merge.go b/pkg/devcontainer/config/merge.go index e6dc58dfb..1f298b502 100644 --- a/pkg/devcontainer/config/merge.go +++ b/pkg/devcontainer/config/merge.go @@ -6,7 +6,7 @@ import ( "github.com/loft-sh/devpod/pkg/types" ) -func MergeConfiguration(config *DevContainerConfig, imageMetadataEntries []*ImageMetadata) (*MergedDevContainerConfig, error) { +func MergeConfiguration(config *Config, imageMetadataEntries []*ImageMetadata) (*MergedConfig, error) { customizations := map[string][]interface{}{} for _, imageMetadata := range imageMetadataEntries { for k, v := range imageMetadata.Customizations { @@ -14,21 +14,21 @@ func MergeConfiguration(config *DevContainerConfig, imageMetadataEntries []*Imag } } - copiedConfig := CloneDevContainerConfig(config) + copiedConfig := CloneConfig(config) // reverse the order reversed := ReverseSlice(imageMetadataEntries) // merge config - mergedConfig := &MergedDevContainerConfig{ + mergedConfig := &MergedConfig{ UpdatedConfigProperties: UpdatedConfigProperties{ Customizations: customizations, }, - DevContainerConfigBase: copiedConfig.DevContainerConfigBase, - NonComposeBase: copiedConfig.NonComposeBase, - ImageContainer: copiedConfig.ImageContainer, - ComposeContainer: copiedConfig.ComposeContainer, - DockerfileContainer: copiedConfig.DockerfileContainer, + ConfigBase: copiedConfig.ConfigBase, + NonComposeBase: copiedConfig.NonComposeBase, + ImageContainer: copiedConfig.ImageContainer, + ComposeContainer: copiedConfig.ComposeContainer, + DockerfileContainer: copiedConfig.DockerfileContainer, } // adjust config diff --git a/pkg/devcontainer/config/metadata.go b/pkg/devcontainer/config/metadata.go index a0278b981..c639180f6 100644 --- a/pkg/devcontainer/config/metadata.go +++ b/pkg/devcontainer/config/metadata.go @@ -6,9 +6,9 @@ type ImageMetadataConfig struct { } type ImageMetadata struct { - ID string `json:"id,omitempty"` - Entrypoint string `json:"entrypoint,omitempty"` - DevContainerConfigBase `json:",inline"` - DevContainerActions `json:",inline"` - NonComposeBase `json:",inline"` + ID string `json:"id,omitempty"` + Entrypoint string `json:"entrypoint,omitempty"` + ConfigBase `json:",inline"` + Actions `json:",inline"` + NonComposeBase `json:",inline"` } diff --git a/pkg/devcontainer/config/parse.go b/pkg/devcontainer/config/parse.go index d02a64290..83701e929 100644 --- a/pkg/devcontainer/config/parse.go +++ b/pkg/devcontainer/config/parse.go @@ -15,13 +15,13 @@ import ( "github.com/tidwall/jsonc" ) -const DEVCONTAINER_FEATURE_FILE_NAME = "devcontainer-feature.json" +const FEATURE_FILE_NAME = "devcontainer-feature.json" -func ParseDevContainerFeature(folder string) (*FeatureConfig, error) { - path := filepath.Join(folder, DEVCONTAINER_FEATURE_FILE_NAME) +func ParseFeature(folder string) (*FeatureConfig, error) { + path := filepath.Join(folder, FEATURE_FILE_NAME) _, err := os.Stat(path) if err != nil { - return nil, fmt.Errorf("%s is missing in feature folder", DEVCONTAINER_FEATURE_FILE_NAME) + return nil, fmt.Errorf("%s is missing in feature folder", FEATURE_FILE_NAME) } path, err = filepath.Abs(path) @@ -44,7 +44,7 @@ func ParseDevContainerFeature(folder string) (*FeatureConfig, error) { return featureConfig, nil } -func SaveDevContainerJSON(config *DevContainerConfig) error { +func Save(config *Config) error { if config.Origin == "" { return fmt.Errorf("no origin in config") } @@ -67,7 +67,7 @@ func SaveDevContainerJSON(config *DevContainerConfig) error { return nil } -func ParseDevContainerJSON(folder, relativePath string) (*DevContainerConfig, error) { +func Parse(folder, relativePath string) (*Config, error) { path := "" if relativePath != "" { path = path2.Join(filepath.ToSlash(folder), relativePath) @@ -103,7 +103,7 @@ func ParseDevContainerJSON(folder, relativePath string) (*DevContainerConfig, er return nil, err } - devContainer := &DevContainerConfig{} + devContainer := &Config{} err = json.Unmarshal(jsonc.ToJSON(bytes), devContainer) if err != nil { return nil, err @@ -113,7 +113,7 @@ func ParseDevContainerJSON(folder, relativePath string) (*DevContainerConfig, er return replaceLegacy(devContainer) } -func replaceLegacy(config *DevContainerConfig) (*DevContainerConfig, error) { +func replaceLegacy(config *Config) (*Config, error) { if len(config.Extensions) == 0 && len(config.Settings) == 0 && config.DevPort == 0 { return config, nil } diff --git a/pkg/devcontainer/config/parse_test.go b/pkg/devcontainer/config/parse_test.go index 59afe0b82..ea2c5428e 100644 --- a/pkg/devcontainer/config/parse_test.go +++ b/pkg/devcontainer/config/parse_test.go @@ -8,7 +8,7 @@ import ( func TestSaveDevContainerJSON(t *testing.T) { type args struct { - config *DevContainerConfig + config *Config } tests := []struct { name string @@ -19,7 +19,7 @@ func TestSaveDevContainerJSON(t *testing.T) { { name: "test omit build field in devcontainer.json", args: args{ - config: &DevContainerConfig{ + config: &Config{ ImageContainer: ImageContainer{ Image: "test", }, @@ -39,7 +39,7 @@ func TestSaveDevContainerJSON(t *testing.T) { tt.args.config.Origin = filepath.Join(tmpDir, "devcontainer.json") - if err := SaveDevContainerJSON(tt.args.config); (err != nil) != tt.wantErr { + if err := Save(tt.args.config); (err != nil) != tt.wantErr { t.Errorf("SaveDevContainerJSON() error = %v, wantErr %v", err, tt.wantErr) } diff --git a/pkg/devcontainer/config/prebuild.go b/pkg/devcontainer/config/prebuild.go index ce351a4e6..eb95d6972 100644 --- a/pkg/devcontainer/config/prebuild.go +++ b/pkg/devcontainer/config/prebuild.go @@ -15,8 +15,8 @@ import ( "github.com/loft-sh/log/hash" ) -func CalculatePrebuildHash(originalConfig *DevContainerConfig, platform, architecture, contextPath, dockerfilePath, dockerfileContent string, log log.Logger) (string, error) { - parsedConfig := CloneDevContainerConfig(originalConfig) +func CalculatePrebuildHash(originalConfig *Config, platform, architecture, contextPath, dockerfilePath, dockerfileContent string, log log.Logger) (string, error) { + parsedConfig := CloneConfig(originalConfig) if platform != "" { splitted := strings.Split(platform, "/") @@ -27,9 +27,9 @@ func CalculatePrebuildHash(originalConfig *DevContainerConfig, platform, archite // delete all options that are not relevant for the build parsedConfig.Origin = "" - parsedConfig.DevContainerActions = DevContainerActions{} + parsedConfig.Actions = Actions{} parsedConfig.NonComposeBase = NonComposeBase{} - parsedConfig.DevContainerConfigBase = DevContainerConfigBase{ + parsedConfig.ConfigBase = ConfigBase{ Name: parsedConfig.Name, Features: parsedConfig.Features, OverrideFeatureInstallOrder: parsedConfig.OverrideFeatureInstallOrder, diff --git a/pkg/devcontainer/config/result.go b/pkg/devcontainer/config/result.go index 4cb8f267e..bd83937e8 100644 --- a/pkg/devcontainer/config/result.go +++ b/pkg/devcontainer/config/result.go @@ -4,14 +4,14 @@ const UserLabel = "devpod.user" type Result struct { DevContainerConfigWithPath *DevContainerConfigWithPath `json:"DevContainerConfigWithPath"` - MergedConfig *MergedDevContainerConfig `json:"MergedConfig"` + MergedConfig *MergedConfig `json:"MergedConfig"` SubstitutionContext *SubstitutionContext `json:"SubstitutionContext"` ContainerDetails *ContainerDetails `json:"ContainerDetails"` } type DevContainerConfigWithPath struct { // Config is the devcontainer.json config - Config *DevContainerConfig `json:"config,omitempty"` + Config *Config `json:"config,omitempty"` // Path is the relative path to the devcontainer.json from the workspace folder Path string `json:"path,omitempty"` @@ -42,7 +42,7 @@ func GetRemoteUser(result *Result) string { return user } -func GetDevPodCustomizations(parsedConfig *DevContainerConfig) *DevPodCustomizations { +func GetDevPodCustomizations(parsedConfig *Config) *DevPodCustomizations { if parsedConfig.Customizations == nil || parsedConfig.Customizations["devpod"] == nil { return &DevPodCustomizations{} } @@ -56,7 +56,7 @@ func GetDevPodCustomizations(parsedConfig *DevContainerConfig) *DevPodCustomizat return devPod } -func GetVSCodeConfiguration(mergedConfig *MergedDevContainerConfig) *VSCodeCustomizations { +func GetVSCodeConfiguration(mergedConfig *MergedConfig) *VSCodeCustomizations { if mergedConfig.Customizations == nil || mergedConfig.Customizations["vscode"] == nil { return &VSCodeCustomizations{} } diff --git a/pkg/devcontainer/config/substitute.go b/pkg/devcontainer/config/substitute.go index 22945d170..1e8f41e8b 100644 --- a/pkg/devcontainer/config/substitute.go +++ b/pkg/devcontainer/config/substitute.go @@ -16,8 +16,8 @@ type ReplaceFunction func(match, variable string, args []string) string var VariableRegExp = regexp.MustCompile(`\${(.*?)}`) type SubstitutedConfig struct { - Config *DevContainerConfig - Raw *DevContainerConfig + Config *Config + Raw *Config } type SubstitutionContext struct { diff --git a/pkg/devcontainer/config/userenvprobe.go b/pkg/devcontainer/config/userenvprobe.go index 35a683520..d1fce6631 100644 --- a/pkg/devcontainer/config/userenvprobe.go +++ b/pkg/devcontainer/config/userenvprobe.go @@ -58,7 +58,7 @@ func ProbeUserEnv(ctx context.Context, probe string, userName string, log log.Lo return map[string]string{}, nil } - preferredShell, err := shell.GetShell(userName) + preferredShell, err := shell.Get(userName) if err != nil { return nil, fmt.Errorf("find shell for user %s: %w", userName, err) } diff --git a/pkg/devcontainer/feature/extend.go b/pkg/devcontainer/feature/extend.go index 5d216d4ff..459435fcf 100644 --- a/pkg/devcontainer/feature/extend.go +++ b/pkg/devcontainer/feature/extend.go @@ -223,7 +223,7 @@ func findContainerUsers(baseImageMetadata *config.ImageMetadataConfig, composeSe return containerUser, remoteUser } -func fetchFeatures(devContainerConfig *config.DevContainerConfig, log log.Logger, forceBuild bool) ([]*config.FeatureSet, error) { +func fetchFeatures(devContainerConfig *config.Config, log log.Logger, forceBuild bool) ([]*config.FeatureSet, error) { featureSets := []*config.FeatureSet{} for featureID, featureOptions := range devContainerConfig.Features { featureFolder, err := ProcessFeatureID(featureID, devContainerConfig, log, forceBuild) @@ -233,7 +233,7 @@ func fetchFeatures(devContainerConfig *config.DevContainerConfig, log log.Logger // parse feature log.Debugf("Parse dev container feature in %s", featureFolder) - featureConfig, err := config.ParseDevContainerFeature(featureFolder) + featureConfig, err := config.ParseFeature(featureFolder) if err != nil { return nil, errors.Wrap(err, "parse feature "+featureID) } @@ -270,7 +270,7 @@ func NormalizeFeatureID(featureID string) string { return ref.String() } -func computeFeatureOrder(devContainer *config.DevContainerConfig, features []*config.FeatureSet) ([]*config.FeatureSet, error) { +func computeFeatureOrder(devContainer *config.Config, features []*config.FeatureSet) ([]*config.FeatureSet, error) { if len(devContainer.OverrideFeatureInstallOrder) == 0 { return computeAutomaticFeatureOrder(features) } diff --git a/pkg/devcontainer/feature/features.go b/pkg/devcontainer/feature/features.go index 1c1e0d55f..0c7725a7c 100644 --- a/pkg/devcontainer/feature/features.go +++ b/pkg/devcontainer/feature/features.go @@ -91,7 +91,7 @@ func escapeQuotesForShell(str string) string { return strings.ReplaceAll(str, "'", `'\''`) } -func ProcessFeatureID(id string, devContainerConfig *config.DevContainerConfig, log log.Logger, forceBuild bool) (string, error) { +func ProcessFeatureID(id string, devContainerConfig *config.Config, log log.Logger, forceBuild bool) (string, error) { if strings.HasPrefix(id, "https://") || strings.HasPrefix(id, "http://") { log.Debugf("Process url feature") return processDirectTarFeature(id, config.GetDevPodCustomizations(devContainerConfig).FeatureDownloadHTTPHeaders, log, forceBuild) @@ -112,7 +112,7 @@ func processOCIFeature(id string, log log.Logger) (string, error) { _, err := os.Stat(featureExtractedFolder) if err == nil { // make sure feature.json is there as well - _, err = os.Stat(filepath.Join(featureExtractedFolder, config.DEVCONTAINER_FEATURE_FILE_NAME)) + _, err = os.Stat(filepath.Join(featureExtractedFolder, config.FEATURE_FILE_NAME)) if err == nil { return featureExtractedFolder, nil } else { diff --git a/pkg/devcontainer/metadata/metadata.go b/pkg/devcontainer/metadata/metadata.go index 2aa59e731..21a8675aa 100644 --- a/pkg/devcontainer/metadata/metadata.go +++ b/pkg/devcontainer/metadata/metadata.go @@ -38,7 +38,7 @@ func GetDevContainerMetadata(substitutionContext *config.SubstitutionContext, ba func FeatureConfigToImageMetadata(feature *config.FeatureConfig) *config.ImageMetadata { return &config.ImageMetadata{ Entrypoint: feature.Entrypoint, - DevContainerActions: config.DevContainerActions{ + Actions: config.Actions{ Customizations: feature.Customizations, }, NonComposeBase: config.NonComposeBase{ @@ -51,9 +51,9 @@ func FeatureConfigToImageMetadata(feature *config.FeatureConfig) *config.ImageMe } } -func DevContainerConfigToImageMetadata(devConfig *config.DevContainerConfig) *config.ImageMetadata { +func DevContainerConfigToImageMetadata(devConfig *config.Config) *config.ImageMetadata { return &config.ImageMetadata{ - DevContainerConfigBase: config.DevContainerConfigBase{ + ConfigBase: config.ConfigBase{ ForwardPorts: devConfig.ForwardPorts, PortsAttributes: devConfig.PortsAttributes, OtherPortsAttributes: devConfig.OtherPortsAttributes, @@ -66,7 +66,7 @@ func DevContainerConfigToImageMetadata(devConfig *config.DevContainerConfig) *co HostRequirements: devConfig.HostRequirements, OverrideCommand: devConfig.OverrideCommand, }, - DevContainerActions: config.DevContainerActions{ + Actions: config.Actions{ OnCreateCommand: devConfig.OnCreateCommand, UpdateContentCommand: devConfig.UpdateContentCommand, PostCreateCommand: devConfig.PostCreateCommand, diff --git a/pkg/devcontainer/run.go b/pkg/devcontainer/run.go index 3d9f02ab9..436859ad2 100644 --- a/pkg/devcontainer/run.go +++ b/pkg/devcontainer/run.go @@ -188,18 +188,18 @@ func (r *runner) recreateCustomDriver(ctx context.Context, options UpOptions, ti return r.Up(ctx, options, timeout) } -func cleanupBuildInformation(c *config.DevContainerConfig) { +func cleanupBuildInformation(c *config.Config) { contextPath := config.GetContextPath(c) _ = os.RemoveAll(filepath.Join(contextPath, config.DevPodContextFeatureFolder)) } -func isDockerFileConfig(config *config.DevContainerConfig) bool { +func isDockerFileConfig(config *config.Config) bool { return config.GetDockerfile() != "" } func runInitializeCommand( workspaceFolder string, - config *config.DevContainerConfig, + config *config.Config, extraEnvVars []string, log log.Logger, ) error { @@ -242,7 +242,7 @@ func runInitializeCommand( func getWorkspace( workspaceFolder, workspaceID string, - conf *config.DevContainerConfig, + conf *config.Config, ) (string, string) { if conf.WorkspaceMount != "" { mount := config.ParseMount(conf.WorkspaceMount) diff --git a/pkg/devcontainer/setup.go b/pkg/devcontainer/setup.go index 383aed00e..45103f766 100644 --- a/pkg/devcontainer/setup.go +++ b/pkg/devcontainer/setup.go @@ -26,19 +26,21 @@ import ( func (r *runner) setupContainer( ctx context.Context, - rawConfig *config.DevContainerConfig, + rawConfig *config.Config, containerDetails *config.ContainerDetails, - mergedConfig *config.MergedDevContainerConfig, + mergedConfig *config.MergedConfig, substitutionContext *config.SubstitutionContext, timeout time.Duration, ) (*config.Result, error) { // inject agent - err := agent.InjectAgent(ctx, func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + execFn := func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { return r.Driver.CommandDevContainer(ctx, r.ID, "root", command, stdin, stdout, stderr) - }, false, agent.ContainerDevPodHelperLocation, agent.DefaultAgentDownloadURL(), false, r.Log, timeout) + } + err := agent.Inject(ctx, execFn, false, agent.DevPodBinary, agent.DefaultAgentDownloadURL(), false, "", nil, nil, nil, r.Log, timeout) if err != nil { return nil, errors.Wrap(err, "inject agent") } + r.Log.Debugf("Injected into container") defer r.Log.Debugf("Done setting up container") @@ -96,14 +98,14 @@ func (r *runner) setupContainer( _, isDockerDriver := r.Driver.(driver.DockerDriver) // ssh tunnel - sshTunnelCmd := fmt.Sprintf("'%s' helper ssh-server --stdio", agent.ContainerDevPodHelperLocation) + sshTunnelCmd := fmt.Sprintf("'%s' helper ssh-server --stdio", agent.DevPodBinary) if r.Log.GetLevel() == logrus.DebugLevel { sshTunnelCmd += " --debug" } // setup container r.Log.Infof("Setup container...") - setupCommand := fmt.Sprintf("'%s' agent container setup --setup-info '%s' --container-workspace-info '%s'", agent.ContainerDevPodHelperLocation, compressed, workspaceConfigCompressed) + setupCommand := fmt.Sprintf("'%s' agent container setup --setup-info '%s' --container-workspace-info '%s'", agent.DevPodBinary, compressed, workspaceConfigCompressed) if runtime.GOOS == "linux" || !isDockerDriver { setupCommand += " --chown-workspace" } diff --git a/pkg/devcontainer/setup/setup.go b/pkg/devcontainer/setup/setup.go index c40d56ae7..d3a1f6348 100644 --- a/pkg/devcontainer/setup/setup.go +++ b/pkg/devcontainer/setup/setup.go @@ -196,7 +196,7 @@ func PatchEtcEnvironmentFlags(workspaceEnv []string, log log.Logger) error { return nil } -func PatchEtcEnvironment(mergedConfig *config.MergedDevContainerConfig, log log.Logger) error { +func PatchEtcEnvironment(mergedConfig *config.MergedConfig, log log.Logger) error { if len(mergedConfig.RemoteEnv) == 0 { return nil } diff --git a/pkg/devcontainer/single.go b/pkg/devcontainer/single.go index 9355c4866..6a083db7b 100644 --- a/pkg/devcontainer/single.go +++ b/pkg/devcontainer/single.go @@ -37,7 +37,7 @@ func (r *runner) runSingleContainer( // does the container already exist? var ( - mergedConfig *config.MergedDevContainerConfig + mergedConfig *config.MergedConfig ) // if options.Recreate is true, and workspace is a running container, we should not rebuild if options.Recreate && parsedConfig.Config.ContainerID != "" { @@ -129,7 +129,7 @@ func (r *runner) runContainer( ctx context.Context, parsedConfig *config.SubstitutedConfig, substitutionContext *config.SubstitutionContext, - mergedConfig *config.MergedDevContainerConfig, + mergedConfig *config.MergedConfig, buildInfo *config.BuildInfo, ) error { var err error @@ -170,7 +170,7 @@ func (r *runner) runContainer( } func (r *runner) getDockerlessRunOptions( - mergedConfig *config.MergedDevContainerConfig, + mergedConfig *config.MergedConfig, substitutionContext *config.SubstitutionContext, buildInfo *config.BuildInfo, ) (*driver.RunOptions, error) { @@ -247,7 +247,7 @@ func (r *runner) getDockerlessRunOptions( } func (r *runner) getRunOptions( - mergedConfig *config.MergedDevContainerConfig, + mergedConfig *config.MergedConfig, substitutionContext *config.SubstitutionContext, buildInfo *config.BuildInfo, ) (*driver.RunOptions, error) { @@ -312,7 +312,7 @@ func (r *runner) addExtraEnvVars(env map[string]string) map[string]string { return env } -func GetStartScript(mergedConfig *config.MergedDevContainerConfig) string { +func GetStartScript(mergedConfig *config.MergedConfig) string { customEntrypoints := mergedConfig.Entrypoints return `echo Container started trap "exit 0" 15 @@ -321,7 +321,7 @@ exec "$@" while sleep 1 & wait $!; do :; done` } -func GetContainerEntrypointAndArgs(mergedConfig *config.MergedDevContainerConfig, imageDetails *config.ImageDetails) (string, []string) { +func GetContainerEntrypointAndArgs(mergedConfig *config.MergedConfig, imageDetails *config.ImageDetails) (string, []string) { cmd := []string{"-c", GetStartScript(mergedConfig), "-"} // `wait $!` allows for the `trap` to run (synchronous `sleep` would not). if imageDetails != nil && mergedConfig.OverrideCommand != nil && !*mergedConfig.OverrideCommand { cmd = append(cmd, imageDetails.Config.Entrypoint...) diff --git a/pkg/driver/docker.go b/pkg/driver/docker.go index 800cb936f..1654e2087 100644 --- a/pkg/driver/docker.go +++ b/pkg/driver/docker.go @@ -25,7 +25,7 @@ type DockerDriver interface { ctx context.Context, workspaceId string, options *RunOptions, - parsedConfig *config.DevContainerConfig, + parsedConfig *config.Config, init *bool, ide string, ideOptions map[string]config2.OptionValue, diff --git a/pkg/driver/docker/docker.go b/pkg/driver/docker/docker.go index 770e61a82..fe873f9d1 100644 --- a/pkg/driver/docker/docker.go +++ b/pkg/driver/docker/docker.go @@ -210,7 +210,7 @@ func (d *dockerDriver) RunDockerDevContainer( ctx context.Context, workspaceId string, options *driver.RunOptions, - parsedConfig *config.DevContainerConfig, + parsedConfig *config.Config, init *bool, ide string, ideOptions map[string]config2.OptionValue, diff --git a/pkg/language/language.go b/pkg/language/language.go index 194866c79..6b2c181a6 100644 --- a/pkg/language/language.go +++ b/pkg/language/language.go @@ -47,7 +47,7 @@ var MapLanguages = map[ProgrammingLanguage]ProgrammingLanguage{ C: Cpp, } -var MapConfig = map[ProgrammingLanguage]*config.DevContainerConfig{ +var MapConfig = map[ProgrammingLanguage]*config.Config{ None: { ImageContainer: config.ImageContainer{ Image: "mcr.microsoft.com/devcontainers/base:ubuntu", @@ -100,7 +100,7 @@ var MapConfig = map[ProgrammingLanguage]*config.DevContainerConfig{ }, } -func DefaultConfig(startPath string, log log.Logger) *config.DevContainerConfig { +func DefaultConfig(startPath string, log log.Logger) *config.Config { language, err := DetectLanguage(startPath) if err != nil { log.Errorf("Error detecting project language: %v", err) diff --git a/pkg/options/resolver/sub_options.go b/pkg/options/resolver/sub_options.go index a5e2c7cef..0b3379338 100644 --- a/pkg/options/resolver/sub_options.go +++ b/pkg/options/resolver/sub_options.go @@ -23,7 +23,7 @@ func execOptionCommand(ctx context.Context, command string, resolvedOptions map[ env = append(env, k+"="+v) } - err := shell.ExecuteCommandWithShell(ctx, command, nil, stdout, stderr, env) + err := shell.Execute(ctx, command, nil, stdout, stderr, env) if err != nil { return nil, errors.Wrapf(err, "exec command: %s%s", stdout.String(), stderr.String()) } diff --git a/pkg/provider/workspace.go b/pkg/provider/workspace.go index 4dbc13652..d72da12dd 100644 --- a/pkg/provider/workspace.go +++ b/pkg/provider/workspace.go @@ -46,7 +46,7 @@ type Workspace struct { DevContainerPath string `json:"devContainerPath,omitempty"` // DevContainerConfig holds the config for the devcontainer.json. - DevContainerConfig *devcontainerconfig.DevContainerConfig `json:"devContainerConfig,omitempty"` + DevContainerConfig *devcontainerconfig.Config `json:"devContainerConfig,omitempty"` // CreationTimestamp is the timestamp when this workspace was created CreationTimestamp types.Time `json:"creationTimestamp,omitempty"` diff --git a/pkg/shell/shell.go b/pkg/shell/shell.go index 0ea716e73..64b51a8f3 100644 --- a/pkg/shell/shell.go +++ b/pkg/shell/shell.go @@ -19,7 +19,8 @@ import ( "mvdan.cc/sh/v3/syntax" ) -func ExecuteCommandWithShell( +// Execute runs a given command using a shell appropriate for the OS, for windows it will emulate a shell +func Execute( ctx context.Context, command string, stdin io.Reader, @@ -49,10 +50,11 @@ func ExecuteCommandWithShell( } // run emulated shell - return RunEmulatedShell(ctx, command, stdin, stdout, stderr, environ) + return RunEmulated(ctx, command, stdin, stdout, stderr, environ) } -func RunEmulatedShell(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer, env []string) error { +// RunEmulated emulates a shell for windows to execute commands in +func RunEmulated(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer, env []string) error { // Let's parse the complete command parsed, err := syntax.NewParser().Parse(strings.NewReader(command), "") if err != nil { @@ -125,7 +127,8 @@ func (devNull) Close() error { return nil } -func GetShell(userName string) ([]string, error) { +// Get returns a list of available shells found on the system +func Get(userName string) ([]string, error) { // try to get a shell if runtime.GOOS != "windows" { // infere login shell from getent diff --git a/pkg/single/single.go b/pkg/single/single.go index 8b367f038..9db238226 100644 --- a/pkg/single/single.go +++ b/pkg/single/single.go @@ -14,6 +14,7 @@ import ( type CreateCommand func() (*exec.Cmd, error) +// Single ensures that certain commands are executed only once at a time, using a file-based locking mechanism func Single(file string, createCommand CreateCommand) error { file = filepath.Join(os.TempDir(), file) fileLock := flock.New(file + ".lock") diff --git a/pkg/ssh/server/ssh.go b/pkg/ssh/server/ssh.go index 2bbe32794..95e242a7c 100644 --- a/pkg/ssh/server/ssh.go +++ b/pkg/ssh/server/ssh.go @@ -24,7 +24,7 @@ import ( var DefaultPort = 8022 func NewServer(addr string, hostKey []byte, keys []ssh.PublicKey, workdir string, log log.Logger) (*Server, error) { - sh, err := shell.GetShell("") + sh, err := shell.Get("") if err != nil { return nil, err } diff --git a/pkg/tunnel/container.go b/pkg/tunnel/container.go index d605b4638..0f18f46c1 100644 --- a/pkg/tunnel/container.go +++ b/pkg/tunnel/container.go @@ -73,7 +73,7 @@ func (c *ContainerHandler) Run(ctx context.Context, handler Handler, cfg *config if c.log.GetLevel() == logrus.DebugLevel { command += " --debug" } - tunnelChan <- agent.InjectAgentAndExecute( + tunnelChan <- agent.Inject( cancelCtx, func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { return c.client.Command(ctx, client.CommandOptions{ diff --git a/pkg/tunnel/forwarder.go b/pkg/tunnel/forwarder.go index 064912e68..4003a231f 100644 --- a/pkg/tunnel/forwarder.go +++ b/pkg/tunnel/forwarder.go @@ -19,6 +19,7 @@ func newForwarder(sshClient *ssh.Client, forwardedPorts []string, log log.Logger } } +// forwarder manages which ports to forward using a ssh client type forwarder struct { m sync.Mutex diff --git a/pkg/tunnel/services.go b/pkg/tunnel/services.go index ef528cb65..3c087e6da 100644 --- a/pkg/tunnel/services.go +++ b/pkg/tunnel/services.go @@ -29,7 +29,7 @@ import ( "k8s.io/client-go/util/retry" ) -func RunInContainer( +func StartCredentialsServer( ctx context.Context, devPodConfig *config.Config, containerClient *ssh.Client, @@ -111,7 +111,7 @@ func RunInContainer( writer := log.ErrorStreamOnly().Writer(logrus.DebugLevel, false) defer writer.Close() - command := fmt.Sprintf("'%s' agent container credentials-server --user '%s'", agent.ContainerDevPodHelperLocation, user) + command := fmt.Sprintf("'%s' agent container credentials-server --user '%s'", agent.DevPodBinary, user) if configureGitCredentials { command += " --configure-git-helper" } diff --git a/pkg/workspace/machine.go b/pkg/workspace/machine.go index 52ab650b8..cca6e4182 100644 --- a/pkg/workspace/machine.go +++ b/pkg/workspace/machine.go @@ -41,6 +41,7 @@ func listMachines(devPodConfig *config.Config, log log.Logger) ([]*provider2.Mac return retMachines, nil } +// ResolveMachine uses the given args to either find or create a machine func ResolveMachine(devPodConfig *config.Config, args []string, userOptions []string, log log.Logger) (client.Client, error) { machineClient, err := resolveMachine(devPodConfig, args, log) if err != nil { diff --git a/pkg/workspace/workspace.go b/pkg/workspace/workspace.go index f7ab960a9..87385c711 100644 --- a/pkg/workspace/workspace.go +++ b/pkg/workspace/workspace.go @@ -66,7 +66,7 @@ func Exists(devPodConfig *config.Config, args []string) string { return workspaceID } -func ListWorkspaces(devPodConfig *config.Config, log log.Logger) ([]*provider2.Workspace, error) { +func List(devPodConfig *config.Config, log log.Logger) ([]*provider2.Workspace, error) { workspaceDir, err := provider2.GetWorkspacesDir(devPodConfig.DefaultContext) if err != nil { return nil, err @@ -91,7 +91,7 @@ func ListWorkspaces(devPodConfig *config.Config, log log.Logger) ([]*provider2.W return retWorkspaces, nil } -func GetWorkspaceName(args []string) string { +func GetName(args []string) string { if len(args) == 0 { return "" } @@ -105,8 +105,8 @@ func GetWorkspaceName(args []string) string { return workspaceID } -// GetWorkspace tries to retrieve an already existing workspace -func GetWorkspace(devPodConfig *config.Config, args []string, changeLastUsed bool, log log.Logger) (client.BaseWorkspaceClient, error) { +// Get tries to retrieve an already existing workspace +func Get(devPodConfig *config.Config, args []string, changeLastUsed bool, log log.Logger) (client.BaseWorkspaceClient, error) { provider, workspace, machine, err := getWorkspace(devPodConfig, args, changeLastUsed, log) if err != nil { return nil, err @@ -149,8 +149,8 @@ func getWorkspace(devPodConfig *config.Config, args []string, changeLastUsed boo return loadExistingWorkspace(workspaceID, devPodConfig, changeLastUsed, log) } -// ResolveWorkspace tries to retrieve an already existing workspace or creates a new one -func ResolveWorkspace( +// Resolve tries to retrieve an already existing workspace or creates a new one +func Resolve( ctx context.Context, devPodConfig *config.Config, ide string, @@ -336,14 +336,8 @@ func createWorkspace( return nil, nil, nil, fmt.Errorf("provider '%s' is not initialized, please make sure to run 'devpod provider use %s' at least once before using this provider", provider.Config.Name, provider.Config.Name) } - // get workspace folder - workspaceFolder, err := provider2.GetWorkspaceDir(devPodConfig.DefaultContext, workspaceID) - if err != nil { - return nil, nil, nil, err - } - // resolve workspace - workspace, err := resolve(ctx, provider, devPodConfig, name, workspaceID, workspaceFolder, source, isLocalPath, sshConfigPath, uid) + workspace, err := resolve(ctx, provider, devPodConfig, name, workspaceID, source, isLocalPath, sshConfigPath, uid) if err != nil { return nil, nil, nil, err } @@ -443,8 +437,7 @@ func resolve( defaultProvider *ProviderWithOptions, devPodConfig *config.Config, name, - workspaceID, - workspaceFolder string, + workspaceID string, source *provider2.WorkspaceSource, isLocalPath bool, sshConfigPath string,