diff --git a/CHANGELOG.md b/CHANGELOG.md index 36dd50b..c75c45d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,35 +4,160 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [8.482.08-2] - 2026-06-24 +### Changed +- Upgrade base-image to v3.24.1-1 + +## [8.482.08-1] - 2026-06-15 +### Changed +- Upgrade OpenJDK to v8.482.08 +- Upgrade base-image to v3.24.0-1 + +## [8.452.09-6] - 2026-03-25 +### Changed +- Upgrade base-image to v3.23.3-6 + +## [8.452.09-5] - 2026-03-10 +### Changed +- Upgrade base-image to v3.23.3-5 + +## [8.452.09-4] - 2026-02-17 +### Fixed +- [#91] Upgrade base-image to 3.23.3-4 + - This fixes a bug in doguctl, to not check the local config if volume is not mounted. + +## [8.452.09-3] - 2026-02-12 +### Security +- [#91] Upgrade base-image to 3.23.3-3 + - [#91] Update doguctl to v0.15.0 to fix [CVE-2025-61732](https://avd.aquasec.com/nvd/2026/CVE-2025-61732) and [CVE-2025-68121](https://avd.aquasec.com/nvd/2026/CVE-2025-68121). + +## [8.452.09-2] - 2025-12-10 +### Changed +- Upgrade base-image to 3.23.0-1 + +## [8.452.09-1] - 2025-12-02 +### Changed +- Upgrade base-image to 3.22.0-5 +- Upgrade Java to 8.452.09 + +## [8u432-1] - 2025-01-02 +### Changed +- [#75] Upgrade base image to 3.21.0-1 +- [#75] Upgrade java to 8.432.06-r0 + +### Security +- [#75] Fixes CVE-2024-45337 + +## [8u402-6] - 2024-09-18 +### Changed +- Relicense to AGPL-3.0-only + +## [8u402-5] - 2024-09-17 +### Changed +- [#56] Update Base Image to v3.20.3-1 + +## [8u402-4] - 2024-08-06 +### Changed +- [#48] Update Base Image to v3.20.2-1 + +### Security +- this release closes CVE-2024-41110 + +## [8u402-3] - 2024-06-26 +### Changed +- [#43] Update Base Image to v3.20.1-2 + +### Security +- this release closes the following CVEs + - CVE-2024-24788 + - CVE-2024-24789 + - CVE-2024-24790 + +## [8u402-2] - 2024-06-25 +### Changed +- Upgrade to base image 3.20.1-1 (#41) + - Contains doguctl v0.11.0 +- Update makefiles to 9.0.5 + +## [8u402-1] - 2024-06-07 +### Changed +- Upgrade to base image 3.19.1-2 (#36) + - Contains doguctl v0.10.0 +- Upgrade to OpenJDK 8.402.06-r0 +## [8u392-1] - 2024-02-23 +### Changed +- Upgrade to OpenJDK 8.392.08-r1 + +## [8u372-1] - 2023-09-20 +### Changed +- Upgrade to base image 3.18.3-1 (#22) +- Upgrade to OpenJDK 8.372-07-r0 (#22) + +## [8u362-1] - 2023-04-21 +### Changed +- Upgrade to base image 3.17.3-2 +- Upgrade to OpenJDK 8.362.09-r1 + +## [8u302-3] - 2022-05-11 +### Added +- apk update and upgrade to Dockerfile + +## [8u302-2] - 2022-05-11 +### Changed +- Upgrade to base image 3.15.3-1 +- Upgrade to OpenJDK 8.302.08-r2 + +## [8u302-1] - 2022-02-04 +### Changed +- Upgrade to base image 3.14.3-1 +- Upgrade to OpenJDK 8.302.08-r1 + +## [8u282-1] ### Added - Add support for additional certificates (#4) - see the [operations docs](docs/operations_en.md) for more information +- Update base image to 3.14.2-2 +- Update to OpenJDK 8.282.08-r1 ## [11.0.11-1] - ### Changed - update to Oracle OpenJDK 11.0.11_p9-r0 - update base image to 3.14.2-1 ## [11.0.5-4] - ### Changed - update base image to 3.11.6-3 ## [11.0.5-3] - ### Changed - update base image to 3.11.6-2 ## [11.0.5-2] - ### Changed - update base image to 3.11.6-1 ## [11.0.5-1] - ### Changed - update to Oracle OpenJDK 11.0.5_p10-r0 - update base image to 3.11.5-1 + +## [8u252-3] +### Changed +- update to Oracle OpenJDK 8.252.09-r0 +- update base image to 3.11.6-3 + +## [8u242-3] +### Changed +- update base image to 3.11.6-2 + +## [8u242-2] +- update to Oracle OpenJDK + +### Changed +- update base image to 3.11.6-1 + +## [8u242-2] +### Changed +- update to Oracle OpenJDK 8.242.08-r0 + diff --git a/Dockerfile b/Dockerfile index 9f9da75..0aa7f55 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,20 @@ -FROM registry.cloudogu.com/official/base:3.14.2-2 +# build arguments, defined in Makefile +ARG BASE_IMAGE_VERSION + +FROM registry.cloudogu.com/official/base:${BASE_IMAGE_VERSION} LABEL maintainer="hello@cloudogu.com" -# build arguments, passed from Makefile ARG JAVA_ALPINE_VERSION -# environment variables ENV \ # default to utf-8 encoding LANG="C.UTF-8" \ # java home - JAVA_HOME="/usr/lib/jvm/java-11-openjdk" \ + JAVA_HOME="/usr/lib/jvm/java-1.8-openjdk" \ # add java binaries to path - PATH="$PATH:/usr/lib/jvm/java-11-openjdk/jre/bin:/usr/lib/jvm/java-11-openjdk/bin" + PATH="$PATH:/usr/lib/jvm/java-1.8-openjdk/jre/bin:/usr/lib/jvm/java-1.8-openjdk/bin" RUN set -x \ - # install java JAVA_ALPINE_VERSION is define in Makefile - && apk add --no-cache openjdk11="${JAVA_ALPINE_VERSION}" + && apk add --no-cache openjdk8="${JAVA_ALPINE_VERSION}" -# copy resources COPY resources/ / diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..b3403f3 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,108 @@ +#!groovy +@Library(['github.com/cloudogu/ces-build-lib@4.3.0', 'github.com/cloudogu/dogu-build-lib@v3.4.2']) +import com.cloudogu.ces.cesbuildlib.* +import com.cloudogu.ces.dogubuildlib.* + +timestamps { + node('sos') { + + properties([ + // Keep only the last x builds to preserve space + buildDiscarder(logRotator(numToKeepStr: '10')), + // Don't run concurrent builds for a branch, because they use the same workspace directory + disableConcurrentBuilds(), + parameters([ + booleanParam(name: 'PublishRelease', description: 'Publish a RELEASE image to the registry.', defaultValue: false), + booleanParam(name: 'PublishPrerelease', description: 'Publish a prerelease image to the registry.', defaultValue: false), + choice(name: 'TrivySeverityLevels', choices: [TrivySeverityLevel.CRITICAL, TrivySeverityLevel.HIGH_AND_ABOVE, TrivySeverityLevel.MEDIUM_AND_ABOVE, TrivySeverityLevel.ALL], description: 'The levels to scan with trivy', defaultValue: TrivySeverityLevel.CRITICAL), + choice(name: 'TrivyStrategy', choices: [TrivyScanStrategy.UNSTABLE, TrivyScanStrategy.FAIL, TrivyScanStrategy.IGNORE], description: 'Define whether the build should be unstable, fail or whether the error should be ignored if any vulnerability was found.', defaultValue: TrivyScanStrategy.UNSTABLE), + ]) + ]) + + Git git = new Git(this, "cesmarvin") + GitHub github = new GitHub(this, git) + Changelog changelog = new Changelog(this) + + stage('Checkout') { + checkout scm + } + + final String javaVersion = sh(returnStdout: true, script: 'awk -F\'=\' \'/^JAVA_VERSION=/{gsub(/"/, "", $2); print $2}\' Makefile').trim() + final String changeCounter = sh(returnStdout: true, script: 'awk -F\'=\' \'/^CHANGE_COUNTER=/{gsub(/"/, "", $2); print $2}\' Makefile').trim() + + final String imageName = sh(returnStdout: true, script: 'awk -F\'=\' \'/^IMAGE_NAME=/{gsub(/"/, "", $2); print $2}\' Makefile').trim() + final String imageVersion = "${javaVersion}-${changeCounter}" + + stage('Lint') { + lintDockerfile() + final String scriptFiles = sh(returnStdout: true, script: "find resources -iname '*.sh' | xargs").trim() + shellCheck(scriptFiles) + } + + stage('Build') { + withCredentials([[$class : 'UsernamePasswordMultiBinding', + credentialsId : "cesmarvin-setup", + usernameVariable: 'TOKEN_ID', + passwordVariable: 'TOKEN_SECRET']]) { + sh "docker login -u ${escapeToken(env.TOKEN_ID)} -p ${escapeToken(env.TOKEN_SECRET)} registry.cloudogu.com" + sh "make build" + sh "docker logout registry.cloudogu.com" + } + } + + stage('Test') { + sh "make unit-test-shell-local" + } + + stage('Trivy scan') { + Trivy trivy = new Trivy(this) + trivy.scanImage("${imageName}:${imageVersion}", params.TrivySeverityLevels, params.TrivyStrategy) + trivy.saveFormattedTrivyReport(TrivyScanFormat.TABLE) + trivy.saveFormattedTrivyReport(TrivyScanFormat.JSON) + trivy.saveFormattedTrivyReport(TrivyScanFormat.HTML) + } + + if (params.PublishPrerelease) { + stage('Publish prerelease') { + withCredentials([[$class : 'UsernamePasswordMultiBinding', + credentialsId : "harborrobotprerelease", + usernameVariable: 'TOKEN_ID', + passwordVariable: 'TOKEN_SECRET']]) { + sh "docker login -u ${escapeToken(env.TOKEN_ID)} -p ${escapeToken(env.TOKEN_SECRET)} registry.cloudogu.com" + sh "make deploy-prerelease" + sh "docker logout registry.cloudogu.com" + } + } + } + + if (params.PublishRelease) { + final String currentTag = sh(returnStdout: true, script: "git tag --points-at HEAD").trim() + final String currentBranch = sh(returnStdout: true, script: "git branch --show-current").trim() + stage('Validate tag') { + if (!git.originTagExists(currentTag)) { + println("Creating missing tag: ${imageVersion}") + git.setTag(imageVersion, "Release ${imageVersion}", 'sos-automat', 'sos@cloudogu.com') + git.push(imageVersion) + } + } + stage('Publish release') { + println("Publishing release at tag: ${currentTag}") + withCredentials([[$class : 'UsernamePasswordMultiBinding', + credentialsId : "cesmarvin-setup", + usernameVariable: 'TOKEN_ID', + passwordVariable: 'TOKEN_SECRET']]) { + sh "docker login -u ${escapeToken(env.TOKEN_ID)} -p ${escapeToken(env.TOKEN_SECRET)} registry.cloudogu.com" + sh "make deploy" + sh "docker logout registry.cloudogu.com" + } + github.createReleaseWithChangelog("${imageVersion}", changelog, currentBranch) + } + } + + } +} + +static def escapeToken(String token) { + token = token.replaceAll("\\\$", '\\\\\\\$') + return token +} diff --git a/LICENSE b/LICENSE index 38f268c..bae94e1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,661 @@ -MIT License - -Copyright (c) 2017 Cloudogu - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. \ No newline at end of file diff --git a/Makefile b/Makefile index d743fcd..0585eab 100644 --- a/Makefile +++ b/Makefile @@ -1,53 +1,54 @@ -JAVA_VERSION="11.0.11" -CHANGE_COUNTER="2" -JAVA_ALPINE_VERSION="11.0.11_p9-r0" -IMAGE_NAME="registry.cloudogu.com/official/java" -IMAGE_TAG="$(JAVA_VERSION)-$(CHANGE_COUNTER)" - -MAKEFILES_VERSION=4.5.0 +JAVA_VERSION=8.482.08 +CHANGE_COUNTER=2 +BASE_IMAGE_VERSION=3.24.1-1 +JAVA_ALPINE_VERSION=8.482.08-r0 +IMAGE_NAME=registry.cloudogu.com/official/java +IMAGE_NAME_PRERELEASE=registry.cloudogu.com/prerelease_official/java +IMAGE_TAG=$(JAVA_VERSION)-$(CHANGE_COUNTER) +# renovate: datasource=github-tags depName=cloudogu/makefiles extractVersion=^v(?.*)$ +MAKEFILES_VERSION=10.9.1 default: build -WORKSPACE=/workspace -BATS_LIBRARY_DIR=$(TARGET_DIR)/bats_libs -TESTS_DIR=./unitTests -BASH_TEST_REPORT_DIR=$(TARGET_DIR)/shell_test_reports -BASH_TEST_REPORTS=$(BASH_TEST_REPORT_DIR)/TestReport-*.xml -BATS_ASSERT=$(BATS_LIBRARY_DIR)/bats-assert -BATS_MOCK=$(BATS_LIBRARY_DIR)/bats-mock -BATS_SUPPORT=$(BATS_LIBRARY_DIR)/bats-support -BATS_FILE=$(BATS_LIBRARY_DIR)/bats-file -BATS_BASE_IMAGE?=bats/bats -BATS_CUSTOM_IMAGE?=cloudogu/bats -BATS_TAG?=1.2.1 - include build/make/variables.mk include build/make/self-update.mk include build/make/clean.mk +include build/make/bats.mk + +TESTS_DIR=./unitTests .PHONY: info info: - @echo "version informations ..." - @echo "Java Version : $(JAVA_VERSION)" - @echo "Change Counter: $(CHANGE_COUNTER)" - @echo "Apk Version : $(JAVA_ALPINE_VERSION)" - @echo "Image Name : $(IMAGE_NAME)" - @echo "Image Tag : $(IMAGE_TAG)" - @echo "Image : $(IMAGE_NAME):$(IMAGE_TAG)" + @echo "version information ..." + @echo "Java version : $(JAVA_VERSION)" + @echo "Package version : $(JAVA_ALPINE_VERSION)" + @echo "Base Image version : $(BASE_IMAGE_VERSION)" + @echo "Image (release) : $(IMAGE_NAME):$(IMAGE_TAG)" + @echo "Image (prerelease) : $(IMAGE_NAME_PRERELEASE):$(IMAGE_TAG)" .PHONY: build build: - docker build --build-arg JAVA_ALPINE_VERSION="$(JAVA_ALPINE_VERSION)" -t "$(IMAGE_NAME):$(IMAGE_TAG)" . + docker build \ + --build-arg "BASE_IMAGE_VERSION=$(BASE_IMAGE_VERSION)" \ + --build-arg "JAVA_ALPINE_VERSION=$(JAVA_ALPINE_VERSION)" \ + -t "$(IMAGE_NAME):$(IMAGE_TAG)" . .PHONY: deploy deploy: build + @echo "Publishing image $(IMAGE_NAME):$(IMAGE_TAG)" docker push "$(IMAGE_NAME):$(IMAGE_TAG)" +.PHONY: deploy-prerelease +deploy-prerelease: build + @echo "Publishing image $(IMAGE_NAME_PRERELEASE):$(IMAGE_TAG)" + docker tag "$(IMAGE_NAME):$(IMAGE_TAG)" "$(IMAGE_NAME_PRERELEASE):$(IMAGE_TAG)" + docker rmi "$(IMAGE_NAME):$(IMAGE_TAG)" + docker push "$(IMAGE_NAME_PRERELEASE):$(IMAGE_TAG)" + .PHONY: shell shell: build docker run --rm -ti "$(IMAGE_NAME):$(IMAGE_TAG)" bash || 0 - .PHONY unit-test-shell: unit-test-shell: unit-test-shell-$(ENVIRONMENT) diff --git a/README.md b/README.md index 0ec8cc7..c32b43c 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,35 @@ [![GitHub license](https://img.shields.io/github/license/cloudogu/java.svg)](https://github.com/cloudogu/java/blob/master/LICENSE) [![GitHub release](https://img.shields.io/github/release/cloudogu/java.svg)](https://github.com/cloudogu/java/releases) -# dogu java docker image +# Base Dogu Java image -official/java is based on official/base, thus inheriting doguctl, bash and other tools. +`official/java` is based on `official/base`, thus inheriting _doguctl_, _bash_ and other tools. ## how to build - make build +Detailed instructions for building and releasing the base image can be found in [container_building.en.md](docs/container_building_en.md). +NOTE: For Java _major_, _minor_ and _update_ version and the _change counter_, see _JAVA_VERSION_, _JAVA_ALPINE_VERSION_ and _CHANGE_COUNTER_ in Makefile. -NOTE: _java major version_ and _java minor/update version_ see ENV statement in Dockerfile +## Why are there branches called `javaXX`? -NOTE: _java major version_, _java minor/update version_ and _cloudogu revision_ should be mentioned on the first line in Dockerfile +Some Dogus need older Java version as their bases, therefore, a set of base Dogus with older Java versions must be maintained in addition to the latest stable Java version. ---- -### What is the Cloudogu EcoSystem? -The Cloudogu EcoSystem is an open platform, which lets you choose how and where your team creates great software. Each service or tool is delivered as a Dogu, a Docker container. Each Dogu can easily be integrated in your environment just by pulling it from our registry. We have a growing number of ready-to-use Dogus, e.g. SCM-Manager, Jenkins, Nexus, SonarQube, Redmine and many more. Every Dogu can be tailored to your specific needs. Take advantage of a central authentication service, a dynamic navigation, that lets you easily switch between the web UIs and a smart configuration magic, which automatically detects and responds to dependencies between Dogus. The Cloudogu EcoSystem is open source and it runs either on-premises or in the cloud. The Cloudogu EcoSystem is developed by Cloudogu GmbH under [MIT License](https://cloudogu.com/license.html). +## What is the Cloudogu EcoSystem? +The Cloudogu EcoSystem is an open platform, which lets you choose how and where your team creates great software. Each service or tool is delivered as a Dogu, a Docker container. Each Dogu can easily be integrated in your environment just by pulling it from our registry. + +We have a growing number of ready-to-use Dogus, e.g. SCM-Manager, Jenkins, Nexus Repository, SonarQube, Redmine and many more. Every Dogu can be tailored to your specific needs. Take advantage of a central authentication service, a dynamic navigation, that lets you easily switch between the web UIs and a smart configuration magic, which automatically detects and responds to dependencies between Dogus. -### How to get in touch? -Want to talk to the Cloudogu team? Need help or support? There are several ways to get in touch with us: +The Cloudogu EcoSystem is open source and it runs either on-premises or in the cloud. The Cloudogu EcoSystem is developed by Cloudogu GmbH under [AGPL-3.0-only](https://spdx.org/licenses/AGPL-3.0-only.html). + +## License +Copyright © 2020 - present Cloudogu GmbH +This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, version 3. +This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. +You should have received a copy of the GNU Affero General Public License along with this program. If not, see https://www.gnu.org/licenses/. +See [LICENSE](LICENSE) for details. -* [Website](https://cloudogu.com) -* [myCloudogu-Forum](https://forum.cloudogu.com/topic/34?ctx=1) -* [Email hello@cloudogu.com](mailto:hello@cloudogu.com) --- -© 2020 Cloudogu GmbH - MADE WITH :heart: FOR DEV ADDICTS. [Legal notice / Impressum](https://cloudogu.com/imprint.html) + +MADE WITH :heart: FOR DEV ADDICTS. [Legal notice / Imprint](https://cloudogu.com/en/imprint/?mtm_campaign=ecosystem&mtm_kwd=imprint&mtm_source=github&mtm_medium=link) diff --git a/build/make/bats.mk b/build/make/bats.mk new file mode 100644 index 0000000..2368044 --- /dev/null +++ b/build/make/bats.mk @@ -0,0 +1,68 @@ +WORKSPACE=/workspace +BATS_LIBRARY_DIR=$(TARGET_DIR)/bats_libs +TESTS_DIR=$(WORKDIR)/batsTests +BASH_TEST_REPORT_DIR=$(TARGET_DIR)/shell_test_reports +BASH_TEST_REPORTS=$(BASH_TEST_REPORT_DIR)/TestReport-*.xml +BATS_ASSERT=$(BATS_LIBRARY_DIR)/bats-assert +BATS_MOCK=$(BATS_LIBRARY_DIR)/bats-mock +BATS_SUPPORT=$(BATS_LIBRARY_DIR)/bats-support +BATS_FILE=$(BATS_LIBRARY_DIR)/bats-file +BATS_BASE_IMAGE?=bats/bats +BATS_CUSTOM_IMAGE?=cloudogu/bats +BATS_TAG?=1.12.0 +BATS_DIR=build/make/bats +BATS_WORKDIR="${WORKDIR}"/"${BATS_DIR}" + +.PHONY unit-test-shell: +unit-test-shell: unit-test-shell-$(ENVIRONMENT) + +$(BATS_ASSERT): + @git clone --depth 1 https://github.com/bats-core/bats-assert $@ + @rm -rf $@/.git + +$(BATS_MOCK): + @git clone --depth 1 https://github.com/grayhemp/bats-mock $@ + @rm -rf $@/.git + +$(BATS_SUPPORT): + @git clone --depth 1 https://github.com/bats-core/bats-support $@ + @rm -rf $@/.git + +$(BATS_FILE): + @git clone --depth 1 https://github.com/bats-core/bats-file $@ + @rm -rf $@/.git + +$(BASH_SRC): + BASH_SRC:=$(shell find "${WORKDIR}" -type f -name "*.sh") + +${BASH_TEST_REPORT_DIR}: $(TARGET_DIR) + @mkdir -p $(BASH_TEST_REPORT_DIR) + +unit-test-shell-ci: $(BASH_SRC) $(BASH_TEST_REPORT_DIR) $(BATS_ASSERT) $(BATS_MOCK) $(BATS_SUPPORT) $(BATS_FILE) + @echo "Test shell units on CI server" + @make unit-test-shell-generic + +unit-test-shell-local: $(BASH_SRC) $(PASSWD) $(ETCGROUP) $(HOME_DIR) buildTestImage $(BASH_TEST_REPORT_DIR) $(BATS_ASSERT) $(BATS_MOCK) $(BATS_SUPPORT) $(BATS_FILE) + @echo "Test shell units locally (in Docker)" + @docker run --rm \ + -v $(HOME_DIR):/home/$(USER) \ + -v $(WORKDIR):$(WORKSPACE) \ + -w $(WORKSPACE) \ + --entrypoint="" \ + $(BATS_CUSTOM_IMAGE):$(BATS_TAG) \ + "${BATS_DIR}"/customBatsEntrypoint.sh make unit-test-shell-generic-no-junit + +unit-test-shell-generic: + @bats --report-formatter junit --formatter junit --output ${BASH_TEST_REPORT_DIR} ${TESTS_DIR} + +unit-test-shell-generic-no-junit: + @bats --report-formatter junit --output ${BASH_TEST_REPORT_DIR} ${TESTS_DIR} + +.PHONY buildTestImage: +buildTestImage: + @echo "Build shell test container" + @cd $(BATS_WORKDIR) && docker build \ + --build-arg=BATS_BASE_IMAGE=${BATS_BASE_IMAGE} \ + --build-arg=BATS_TAG=${BATS_TAG} \ + -t ${BATS_CUSTOM_IMAGE}:${BATS_TAG} \ + . \ No newline at end of file diff --git a/build/make/bats/Dockerfile b/build/make/bats/Dockerfile new file mode 100644 index 0000000..95a2787 --- /dev/null +++ b/build/make/bats/Dockerfile @@ -0,0 +1,9 @@ +ARG BATS_BASE_IMAGE +ARG BATS_TAG + +FROM ${BATS_BASE_IMAGE:-bats/bats}:${BATS_TAG:-1.12.0} + +# Make bash more findable by scripts and tests +RUN apk add make git bash +# suppress git "detected dubious ownership" error/warning for repos which are checked out later +RUN git config --global --add safe.directory /workspace \ No newline at end of file diff --git a/build/make/bats/customBatsEntrypoint.sh b/build/make/bats/customBatsEntrypoint.sh new file mode 100755 index 0000000..bfc192b --- /dev/null +++ b/build/make/bats/customBatsEntrypoint.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + +targetReportDir="${PWD}"/target/shell_test_reports +uidgid=1000:1000 +exitcode=0 +"$@" || exitcode=$? +echo "Resetting file ownership to ${uidgid} in ${targetReportDir}/" +chown -R ${uidgid} "${targetReportDir}"/* +echo "exiting with code ${exitcode}" +exit ${exitcode} \ No newline at end of file diff --git a/build/make/bower.mk b/build/make/bower.mk index 1289b70..a2c76a9 100644 --- a/build/make/bower.mk +++ b/build/make/bower.mk @@ -1,7 +1,9 @@ +##@ Bower dependency management + BOWER_JSON=$(WORKDIR)/bower.json .PHONY: bower-install -bower-install: $(BOWER_TARGET) +bower-install: $(BOWER_TARGET) ## Execute yarn run bower (in Docker) ifeq ($(ENVIRONMENT), ci) @@ -19,7 +21,7 @@ $(BOWER_TARGET): $(BOWER_JSON) $(PASSWD) $(YARN_TARGET) -v $(PASSWD):/etc/passwd:ro \ -v $(WORKDIR):$(WORKDIR) \ -w $(WORKDIR) \ - node:8 \ + node:$(NODE_VERSION) \ yarn run bower @touch $@ diff --git a/build/make/build.mk b/build/make/build.mk index 083ec07..c2f512c 100644 --- a/build/make/build.mk +++ b/build/make/build.mk @@ -1,7 +1,9 @@ +##@ Compiling go software + ADDITIONAL_LDFLAGS?=-extldflags -static LDFLAGS?=-ldflags "$(ADDITIONAL_LDFLAGS) -X main.Version=$(VERSION) -X main.CommitID=$(COMMIT_ID)" GOIMAGE?=golang -GOTAG?=1.14.13 +GOTAG?=1.25 GOOS?=linux GOARCH?=amd64 PRE_COMPILE?= @@ -10,9 +12,9 @@ CUSTOM_GO_MOUNT?=-v /tmp:/tmp GO_BUILD_FLAGS?=-mod=vendor -a -tags netgo $(LDFLAGS) -installsuffix cgo -o $(BINARY) .PHONY: compile -compile: $(BINARY) +compile: $(BINARY) ## Compile the go program via Docker -compile-ci: +compile-ci: ## Compile the go program without Docker @echo "Compiling (CI)..." make compile-generic diff --git a/build/make/clean.mk b/build/make/clean.mk index 4f11678..119387f 100644 --- a/build/make/clean.mk +++ b/build/make/clean.mk @@ -1,10 +1,13 @@ +##@ Cleaning + .PHONY: clean -clean: $(ADDITIONAL_CLEAN) +clean: $(ADDITIONAL_CLEAN) ## Remove target and tmp directories rm -rf ${TARGET_DIR} rm -rf ${TMP_DIR} + rm -rf ${UTILITY_BIN_PATH} .PHONY: dist-clean -dist-clean: clean +dist-clean: clean ## Remove all generated directories rm -rf node_modules rm -rf public/vendor rm -rf vendor diff --git a/build/make/coder-lib.sh b/build/make/coder-lib.sh new file mode 100755 index 0000000..2bcc740 --- /dev/null +++ b/build/make/coder-lib.sh @@ -0,0 +1,182 @@ +#!/bin/bash +# a collection of helpful functions to update coder workspaces for rapid development +set -e -u -x -o pipefail + +function getContainerBin() { + if [ -x "$(command -v podman)" ]; then + echo "podman"; + else + echo "docker"; + fi +} + +function getCoderUser() { + # check if coder is installed, so that there is no problem with build and release targets if this is called before + if [ -x "$(command -v coder)" ]; then + coder users show me -o json | jq -r '.username'; + fi +} + +function getAllWorkspaces() { + coder list -c workspace | tail -n+2 +} + +function doesWorkspaceExist() { + coderUser="$1" + workspaceName="$2" + + workspace=$(coder list -a -o json | jq -r "select(.[].owner_name == \"${coderUser}\" and .[].name == \"${workspaceName}\") | .[0].name") + if [ -z "$workspace" ]; then + return 1 #workspace does not exist + else + return 0 + fi +} + +function generateUniqueWorkspaceName() { + local wantedWorkspacePrefix="$1" + # use time to make name unique + local time + time=$(date +'%H-%M-%S') + local lengthOfTime=${#time} + local delimiter='-' + local lengthOfDelimiter=${#delimiter} + # trim prefix, as workspace names are limited to 32 chars + local trimmedPrefix="${wantedWorkspacePrefix:0:$((32 - lengthOfDelimiter - lengthOfTime))}" + local uniqueName="${trimmedPrefix}${delimiter}${time}" + # '--' is forbidden in coder, replace multiple '-' with a single one. + echo "${uniqueName}" | awk '{gsub(/[-]+/,"-")}1' + # returns sth like 'myPrefix-12-45-23' +} + +function buildImage() { + local tag="$1" + local containerBuildDir="${2:-./container}" + local secretDir="${3:-./secrets}" + local containerExec="${4:-podman}" + + # include build-secrets if there are any + local secretArgs=() + if [ -d "$secretDir" ]; then + # shellcheck disable=SC2231 + for secretPath in $secretDir/*; do + # do not match .sh scripts + [[ $secretPath == *.sh ]] && continue + local secretName + secretName=$(basename "$secretPath") + secretArgs+=("--secret=id=$secretName,src=$secretDir/$secretName") + done + fi + + if [ "$containerExec" = "podman" ]; then + $containerExec build -t "$tag" --pull=newer "$containerBuildDir" "${secretArgs[@]}" + else + $containerExec build -t "$tag" --pull "$containerBuildDir" "${secretArgs[@]}" + fi +} + +function doTrivyConvert() { + local trivyFlags=$1 + local outputFile=$2 + local containerExec=$3 + local jsonScanToConvert=$4 + + local containerJsonScanFile="/tmp/scan.json" + + # shellcheck disable=SC2086 + # as globbing is what we want here + "$containerExec" run --rm --pull=always \ + -v trivy-cache:/root/.cache \ + -v "$jsonScanToConvert:$containerJsonScanFile" \ + "${TRIVY_IMAGE}" -q \ + convert $trivyFlags "$containerJsonScanFile" > "$outputFile" +} + +function uploadTemplate() { + local templateDir="${1:?"Error. you need to add the template directory as the first parameter"}" + local templateName="${2:?"Error. you need to add the template name as the second parameter"}" + # for terraform variables (not editable by workspace users) + local variablesFile="${templateDir}/variables.yaml" + if [ -f "$variablesFile" ]; then + local doesVariablesFileExist=1 + fi + if ! coder template push -y -d "$templateDir" ${doesVariablesFileExist:+--variables-file "$variablesFile"} "$templateName"; then + # if template does not exist yet, create it in coder + coder template create -y -d "$templateDir" ${doesVariablesFileExist:+--variables-file "$variablesFile"} "$templateName" + fi +} + +function createNewWorkspace() { + local templateName="$1" + local workspaceName="$2" + # 3. param is optional, set it to autofill prompts for coder params + local templateDir="${3-unset}" + local richParametersFile="${templateDir}/rich-parameters.yaml" + if [ -n "${templateDir+x}" ] && [ -f "$richParametersFile" ]; then + local doesRichParametersFileExist=1 + fi + coder create -t "$templateName" -y "$workspaceName" ${doesRichParametersFileExist:+--rich-parameter-file "$richParametersFile"} +} + +function removeAllOtherWorkspaces() { + local CODER_USER="$1" + local WORKSPACE_PREFIX="$2" + local IGNORED_WORKSPACE="$3" + WORKSPACES="$(getAllWorkspaces)" + for ws in $WORKSPACES; do + if [ "$ws" != "$CODER_USER/$IGNORED_WORKSPACE" ] && [[ "$ws" =~ ^"$CODER_USER/$WORKSPACE_PREFIX" ]]; then + echo "delete $ws" + if ! coder delete "$ws" -y; then + #do it twice as podman always throws an error at the first time + coder delete "$ws" -y + fi + fi + done +} + +function updateWorkspace() { + local coderUser="$1" + local workspaceName="$2" + local qualifiedWorkspaceName="$coderUser/$workspaceName" + if ! coder stop "$qualifiedWorkspaceName" -y; then + #do it twice as podman always throws an error at the first time + coder stop "$qualifiedWorkspaceName" -y + fi + coder update "$qualifiedWorkspaceName" +} + +function startTestWorkspace() { + local coderUser="$1" + local templateDir="$2" + local workspacePrefix="$3" + local templateName="$4" + local reuseTestWorkspace="$5" + + local newWorkspaceName + if [ "$reuseTestWorkspace" = false ]; then + newWorkspaceName="$(generateUniqueWorkspaceName "$workspacePrefix")" + # do that before deleting others, so that i don't need to wait + createNewWorkspace "$templateName" "$newWorkspaceName" "$templateDir" + # trim prefix as the name of the workspace can also get trimmed + removeAllOtherWorkspaces "$coderUser" "${workspacePrefix:0:22}" "$newWorkspaceName" + else + newWorkspaceName="$workspacePrefix" + if ! doesWorkspaceExist "$coderUser" "$newWorkspaceName"; then + createNewWorkspace "$templateName" "$newWorkspaceName" "$templateDir" + else + updateWorkspace "$coderUser" "$newWorkspaceName" + fi + fi +} + +function uploadToNexus() { + local fileToUpload="$1" + local fileNameNexus="${fileToUpload##*/}" + local templateName="$2" + local releaseVersion="$3" + local nexusUrl="${4:-https://ecosystem.cloudogu.com/nexus/repository/itz-bund/coder}" + set +x #disable command printing because of the password + curl --progress-bar -u "$(cat secrets/nexus-user):$(cat secrets/nexus-pw)" --upload-file "$fileToUpload" \ + "$nexusUrl/$templateName/$releaseVersion/$fileNameNexus" + set -x +} \ No newline at end of file diff --git a/build/make/coder.mk b/build/make/coder.mk new file mode 100644 index 0000000..5a82d6a --- /dev/null +++ b/build/make/coder.mk @@ -0,0 +1,163 @@ +SHELL := /bin/bash + +IMAGE_TAG?=${IMAGE_REGISTRY}/coder/coder-${TEMPLATE_NAME}:${VERSION} +REUSE_TEST_WORKSPACE?=false + +#BUILD_DIR given via variables.mk +TEMPLATE_DIR=${WORKDIR}/template +CONTAINER_BUILD_DIR=${WORKDIR}/container +SECRETS_DIR=${WORKDIR}/secrets +CODER_LIB_PATH=${BUILD_DIR}/make/coder-lib.sh + +RELEASE_DIR=${WORKDIR}/release +MAKE_CHANGE_TOKEN_DIR=${RELEASE_DIR}/make +CONTAINER_FILE?=${CONTAINER_BUILD_DIR}/Dockerfile +CONTAINER_IMAGE_CHANGE_TOKEN?=${MAKE_CHANGE_TOKEN_DIR}/${TEMPLATE_NAME}_image_id.txt +CONTAINER_IMAGE_TAR?=${RELEASE_DIR}/${TEMPLATE_NAME}.tar +CONTAINER_IMAGE_TARGZ?=${RELEASE_DIR}/${TEMPLATE_NAME}.tar.gz +CONTAINER_IMAGE_TRIVY_SCAN_JSON?=${RELEASE_DIR}/trivy.json +CONTAINER_IMAGE_TRIVY_SCAN_TABLE?=${RELEASE_DIR}/trivy.txt +CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE?=${RELEASE_DIR}/trivy_critical.txt +CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON?=${RELEASE_DIR}/trivy_critical.json + +IMAGE_REGISTRY?=registry.cloudogu.com +IMAGE_REGISTRY_USER_FILE?=${SECRETS_DIR}/harbor-user +IMAGE_REGISTRY_PW_FILE?=${SECRETS_DIR}/harbor-pw + +CHANGELOG_FILE=${WORKDIR}/CHANGELOG.md +TEMPLATE_RELEASE_TAR_GZ=${RELEASE_DIR}/${TEMPLATE_NAME}-template.tar.gz + +TEST_WORKSPACE_PREFIX?=test-${TEMPLATE_NAME} +CODER_USER?=$(shell . ${CODER_LIB_PATH} && getCoderUser) + +CONTAINER_BIN?=$(shell . ${CODER_LIB_PATH} && getContainerBin) +GOPASS_BIN?=$(shell command -v gopass 2> /dev/null) + +EXCLUDED_TEMPLATE_FILES?=rich-parameters.yaml variables.yaml + +TRIVY_VERSION ?= latest +TRIVY_IMAGE = aquasec/trivy:$(TRIVY_VERSION) + +export TRIVY_IMAGE + +##@ Coder template development + +${SECRETS_DIR}: + mkdir -p ${SECRETS_DIR} + +${IMAGE_REGISTRY_USER_FILE}: ${SECRETS_DIR} +ifeq ($(ENVIRONMENT), local) + @echo "Found developer environment. creating secret ${IMAGE_REGISTRY_USER_FILE}" + @${GOPASS_BIN} show ces/websites/registry.cloudogu.com/robot_coder_jenkins | tail -n 1 | sed -e "s/^username: //" > ${IMAGE_REGISTRY_USER_FILE}; +else + @echo "Found CI environment. Please create secrets yourself" +endif + +${IMAGE_REGISTRY_PW_FILE}: ${SECRETS_DIR} +ifeq ($(ENVIRONMENT), local) + @echo "Found developer environment. creating secret ${IMAGE_REGISTRY_PW_FILE}" + @${GOPASS_BIN} show ces/websites/registry.cloudogu.com/robot_coder_jenkins | head -n 1 > ${IMAGE_REGISTRY_PW_FILE}; +else + @echo "Found CI environment. Please create secrets yourself" +endif + +.PHONY: loadGopassSecrets +loadGopassSecrets: ${IMAGE_REGISTRY_USER_FILE} ${IMAGE_REGISTRY_PW_FILE} ${ADDITIONAL_SECRETS_TARGET} ## load secrets from gopass into secret files, so that the build process works locally + +.PHONY: imageRegistryLogin +imageRegistryLogin: loadGopassSecrets ${IMAGE_REGISTRY_USER_FILE} ${IMAGE_REGISTRY_PW_FILE} ## log in to the registry + @${CONTAINER_BIN} login -u "$$(cat ${IMAGE_REGISTRY_USER_FILE})" --password-stdin '${IMAGE_REGISTRY}' < ${IMAGE_REGISTRY_PW_FILE} + +.PHONY: imageRegistryLogout +imageRegistryLogout: ## log out of the registry + @${CONTAINER_BIN} logout '${IMAGE_REGISTRY}' + +.PHONY: buildImage +buildImage: buildImage-$(ENVIRONMENT) ## build the container image + +.PHONY: buildImage-local +buildImage-local: imageRegistryLogin ${CONTAINER_IMAGE_CHANGE_TOKEN} ## build the container image locally + @echo "if the build is not triggered without a change in the dockerfile, try to delete ${CONTAINER_IMAGE_CHANGE_TOKEN}" + +.PHONY: buildImage-ci +buildImage-ci: ${CONTAINER_IMAGE_CHANGE_TOKEN} ## build the container image without automatic secret management + +${CONTAINER_IMAGE_CHANGE_TOKEN}: ${CONTAINER_FILE} + @. ${CODER_LIB_PATH} && buildImage ${IMAGE_TAG} ${CONTAINER_BUILD_DIR} ${SECRETS_DIR} ${CONTAINER_BIN} + @mkdir -p ${MAKE_CHANGE_TOKEN_DIR} + @${CONTAINER_BIN} image ls --format="{{.ID}}" ${IMAGE_TAG} > ${CONTAINER_IMAGE_CHANGE_TOKEN} + +.PHONY: uploadTemplate +uploadTemplate: ## upload template to coder server + @. ${CODER_LIB_PATH} && uploadTemplate ${TEMPLATE_DIR} ${TEMPLATE_NAME} + +.PHONY: startTestWorkspace +startTestWorkspace: ## start a test workspace with coder + @. ${CODER_LIB_PATH} && startTestWorkspace ${CODER_USER} ${TEMPLATE_DIR} ${TEST_WORKSPACE_PREFIX} ${TEMPLATE_NAME} ${REUSE_TEST_WORKSPACE} + +.PHONY: createImageRelease +createImageRelease: ${CONTAINER_IMAGE_TARGZ} ## export the container image as a tar.gz + +${CONTAINER_IMAGE_TAR}: ${CONTAINER_IMAGE_CHANGE_TOKEN} + ${CONTAINER_BIN} save "${IMAGE_TAG}" -o ${CONTAINER_IMAGE_TAR} + +${CONTAINER_IMAGE_TARGZ}: ${CONTAINER_IMAGE_TAR} + gzip -f --keep "${CONTAINER_IMAGE_TAR}" + +.PHONY: trivyscanImage +trivyscanImage: ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} ${CONTAINER_IMAGE_TRIVY_SCAN_TABLE} ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE} ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON} ## do a trivy scan for the workspace image in various output formats + +${CONTAINER_IMAGE_TRIVY_SCAN_JSON}: ${CONTAINER_IMAGE_TAR} + ${CONTAINER_BIN} run --rm --pull=always \ + -v "trivy-cache:/root/.cache" \ + -v "${CONTAINER_IMAGE_TAR}:/tmp/image.tar" \ + $(TRIVY_IMAGE) -q \ + image --scanners vuln --input /tmp/image.tar -f json --timeout 15m \ + > ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} + +${CONTAINER_IMAGE_TRIVY_SCAN_TABLE}: ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} + @. ${CODER_LIB_PATH} && \ + doTrivyConvert "--format table" ${CONTAINER_IMAGE_TRIVY_SCAN_TABLE} ${CONTAINER_BIN} ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} + +${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE}: ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} + @. ${CODER_LIB_PATH} && \ + doTrivyConvert "--format table --severity CRITICAL" ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE} ${CONTAINER_BIN} ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} + +${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON}: ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} + @. ${CODER_LIB_PATH} && \ + doTrivyConvert "--format json --severity CRITICAL" ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON} ${CONTAINER_BIN} ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} + +.PHONY: createTemplateRelease +createTemplateRelease: ## generate template.tar.gz with all files needed for customers + # remove release dir first as 'cp' cannot merge and will place the source dir inside the target dir if it already exists + rm -rf "${RELEASE_DIR}/${TEMPLATE_NAME}" + cp -r "${TEMPLATE_DIR}" "${RELEASE_DIR}/${TEMPLATE_NAME}/" + #copy changelog + cp "${CHANGELOG_FILE}" "${RELEASE_DIR}/${TEMPLATE_NAME}/" + # remove excludes + for file in "${EXCLUDED_TEMPLATE_FILES}"; do \ + rm -f "${RELEASE_DIR}/${TEMPLATE_NAME}/$$file"; \ + done + tar -czf "${RELEASE_DIR}/${TEMPLATE_NAME}-template.tar.gz" -C "${RELEASE_DIR}" "${TEMPLATE_NAME}" + +.PHONY: createRelease ## generate template- and container archives and the trivy scans +createRelease: createTemplateRelease ${CONTAINER_IMAGE_TARGZ} trivyscanImage ## create the image.tar.gz, template.tar.gz and trivy scans + +.PHONY: cleanCoderRelease +cleanCoderRelease: ## clean release directory + rm -rf "${RELEASE_DIR}" + mkdir -p "${RELEASE_DIR}" + +.PHONY: pushImage +pushImage: ## push the container image into the registry + ${CONTAINER_BIN} push ${IMAGE_TAG} + +.PHONY: uploadRelease +uploadRelease: createTemplateRelease ${CONTAINER_IMAGE_TARGZ} ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} ${CONTAINER_IMAGE_TRIVY_SCAN_TABLE} ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE} ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON} ## upload release artifacts to nexus + @. ${CODER_LIB_PATH} && uploadToNexus ${TEMPLATE_RELEASE_TAR_GZ} ${TEMPLATE_NAME} ${VERSION} + @. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TRIVY_SCAN_JSON} ${TEMPLATE_NAME} ${VERSION} + @. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TRIVY_SCAN_TABLE} ${TEMPLATE_NAME} ${VERSION} + @. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_TABLE} ${TEMPLATE_NAME} ${VERSION} + @. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TRIVY_SCAN_CRITICAL_JSON} ${TEMPLATE_NAME} ${VERSION} + @. ${CODER_LIB_PATH} && uploadToNexus ${CONTAINER_IMAGE_TARGZ} ${TEMPLATE_NAME} ${VERSION} + diff --git a/build/make/dependencies-glide.mk b/build/make/dependencies-glide.mk deleted file mode 100644 index 87eb43f..0000000 --- a/build/make/dependencies-glide.mk +++ /dev/null @@ -1,24 +0,0 @@ -GLIDE=$(GOPATH)/bin/glide -GLIDEFLAGS= -GLIDEHOME=$(GLIDE_HOME) - -ifeq ($(ENVIRONMENT), ci) - GLIDEFLAGS+=--no-color - GLIDEHOME=$(WORKDIR)/.glide_home - GLIDEFLAGS+= --home $(GLIDEHOME) -endif - -.PHONY: update-dependencies -update-dependencies: $(GLIDE) - -.PHONY: dependencies -dependencies: vendor - -vendor: $(GLIDE) glide.yaml glide.lock - @echo "Installing dependencies using Glide..." - $(GLIDE) $(GLIDEFLAGS) install -v - -$(GLIDE): - @echo "installing glide" - @curl https://glide.sh/get | sh - diff --git a/build/make/dependencies-godep.mk b/build/make/dependencies-godep.mk deleted file mode 100644 index 455a69d..0000000 --- a/build/make/dependencies-godep.mk +++ /dev/null @@ -1,10 +0,0 @@ -GODEP=$(GOPATH)/bin/dep - -$(GODEP): - @curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - -vendor: $(GODEP) Gopkg.toml Gopkg.lock - @echo "Installing dependencies using go dep..." - @dep ensure - -dependencies: vendor diff --git a/build/make/dependencies-gomod.mk b/build/make/dependencies-gomod.mk index 1a71d6d..3b3b989 100644 --- a/build/make/dependencies-gomod.mk +++ b/build/make/dependencies-gomod.mk @@ -1,5 +1,7 @@ +##@ Go mod dependency management + .PHONY: dependencies -dependencies: vendor +dependencies: vendor ## Install dependencies using go mod vendor: go.mod go.sum @echo "Installing dependencies using go modules..." diff --git a/build/make/deploy-debian.mk b/build/make/deploy-debian.mk index dd15197..89b0fbe 100644 --- a/build/make/deploy-debian.mk +++ b/build/make/deploy-debian.mk @@ -1,3 +1,5 @@ +##@ Debian package deployment + # This Makefile holds all targets for deploying and undeploying # Uses the variable APT_REPO to determine which apt repos should be used to deploy @@ -40,7 +42,7 @@ else endif .PHONY: deploy -deploy: add-package-to-repo publish +deploy: add-package-to-repo publish ## Deploy package to apt repository define aptly_undeploy PREF=$$(${APTLY} "${APT_API_BASE_URL}/repos/$(1)/packages?q=${ARTIFACT_ID}%20(${VERSION})"); \ @@ -56,8 +58,8 @@ else endif .PHONY: undeploy -undeploy: deploy-check remove-package-from-repo publish +undeploy: deploy-check remove-package-from-repo publish ## Undeploy package from apt repository .PHONE: lint-deb-package -lint-deb-package: debian +lint-deb-package: debian ## Lint debian package @lintian -i $(DEBIAN_PACKAGE) diff --git a/build/make/digital-signature.mk b/build/make/digital-signature.mk index dd005fd..c0eba35 100644 --- a/build/make/digital-signature.mk +++ b/build/make/digital-signature.mk @@ -1,7 +1,9 @@ +##@ Digital signatures + CHECKSUM=$(TARGET_DIR)/$(ARTIFACT_ID).sha256sum .PHONY: checksum -checksum: $(CHECKSUM) +checksum: $(CHECKSUM) ## Generate checksums # we have to depend on target dir, because we want to rebuild the checksum # if one of the artefacts was changed $(CHECKSUM): $(TARGET_DIR) @@ -11,7 +13,12 @@ $(CHECKSUM): $(TARGET_DIR) SIGNATURE=$(CHECKSUM).asc .PHONY: signature -signature: $(SIGNATURE) +signature: $(SIGNATURE) ## Generate signature $(SIGNATURE): $(CHECKSUM) @echo "Generating Signature" @gpg --batch --yes --detach-sign --armor -o $@ $< + +.PHONY: signature-ci +signature-ci: $(CHECKSUM) + @echo "Generating Signature" + @gpg2 --batch --pinentry-mode loopback --passphrase="${passphrase}" --yes --detach-sign --armor -o ${SIGNATURE} $< diff --git a/build/make/info.mk b/build/make/info.mk deleted file mode 100644 index e402a37..0000000 --- a/build/make/info.mk +++ /dev/null @@ -1,8 +0,0 @@ -.PHONY: info -info: - @echo "dumping build information ..." - @echo "Version : $(VERSION)" - @echo "Commit-ID : $(COMMIT_ID)" - @echo "Environment: $(ENVIRONMENT)" - @echo "Branch : $(BRANCH)" - @echo "Packages : $(PACKAGES)" diff --git a/build/make/k8s-component.mk b/build/make/k8s-component.mk new file mode 100644 index 0000000..7eaa439 --- /dev/null +++ b/build/make/k8s-component.mk @@ -0,0 +1,160 @@ +COMPONENT_ARTIFACT_ID?=$(ARTIFACT_ID) +COMPONENT_BUILD_VERSION := $(shell date +%s) +COMPONENT_DEV_VERSION?=${VERSION}-dev.${COMPONENT_BUILD_VERSION} + +ifeq (${K8S_MK_INCLUDE_MARKER}, ) + include ${BUILD_DIR}/make/k8s.mk +endif + +ifeq (${RUNTIME_ENV}, local) + BINARY_HELM_ADDITIONAL_PUSH_ARGS?=--plain-http +endif +BINARY_HELM_ADDITIONAL_PACK_ARGS?= +BINARY_HELM_ADDITIONAL_UNINST_ARGS?= +BINARY_HELM_ADDITIONAL_UPGR_ARGS?= + +HELM_TARGET_DIR ?= $(K8S_RESOURCE_TEMP_FOLDER)/helm +HELM_SOURCE_DIR ?= k8s/helm +HELM_RELEASE_TGZ=${HELM_TARGET_DIR}/${COMPONENT_ARTIFACT_ID}-${VERSION}.tgz +HELM_DEV_RELEASE_TGZ=${HELM_TARGET_DIR}/${COMPONENT_ARTIFACT_ID}-${COMPONENT_DEV_VERSION}.tgz +HELM_ARTIFACT_NAMESPACE?=k8s +ifeq (${RUNTIME_ENV}, remote) + HELM_ARTIFACT_NAMESPACE=testing/k8s +endif +$(info HELM_ARTIFACT_NAMESPACE=$(HELM_ARTIFACT_NAMESPACE)) + +K8S_RESOURCE_COMPONENT ?= "${K8S_RESOURCE_TEMP_FOLDER}/component-${COMPONENT_ARTIFACT_ID}-${VERSION}.yaml" +K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML ?= $(BUILD_DIR)/make/k8s-component.tpl +# HELM_PRE_GENERATE_TARGETS allows to execute targets that affect Helm source files AND Helm target files. +HELM_PRE_GENERATE_TARGETS ?= +# HELM_POST_GENERATE_TARGETS allows to execute targets that only affect Helm target files. +HELM_POST_GENERATE_TARGETS ?= +HELM_PRE_APPLY_TARGETS ?= +COMPONENT_PRE_APPLY_TARGETS ?= + +# This can be used by components with own images to build and push to the dev registry. +# These components should override this variable with `image-import`. +IMAGE_IMPORT_TARGET?= + +##@ K8s - Helm general +.PHONY: helm-init-chart +helm-init-chart: ${BINARY_HELM} ## Creates a Chart.yaml-template with zero values + @echo "Initialize ${HELM_SOURCE_DIR}/Chart.yaml..." + @mkdir -p ${HELM_SOURCE_DIR}/tmp/ + @${BINARY_HELM} create ${HELM_SOURCE_DIR}/tmp/${COMPONENT_ARTIFACT_ID} + @cp ${HELM_SOURCE_DIR}/tmp/${COMPONENT_ARTIFACT_ID}/Chart.yaml ${HELM_SOURCE_DIR}/ + @rm -dr ${HELM_SOURCE_DIR}/tmp + @sed -i 's/appVersion: ".*"/appVersion: "0.0.0-replaceme"/' ${HELM_SOURCE_DIR}/Chart.yaml + @sed -i 's/version: .*/version: 0.0.0-replaceme/' ${HELM_SOURCE_DIR}/Chart.yaml + +.PHONY: helm-generate +helm-generate: ${HELM_TARGET_DIR}/Chart.yaml ${HELM_POST_GENERATE_TARGETS} ## Generates the final helm chart. + +# this is phony because of it is easier this way than the makefile-single-run way +.PHONY: ${HELM_TARGET_DIR}/Chart.yaml +${HELM_TARGET_DIR}/Chart.yaml: $(K8S_RESOURCE_TEMP_FOLDER) validate-chart ${HELM_PRE_GENERATE_TARGETS} copy-helm-files + @echo "Generate Helm chart..." + @if [[ ${STAGE} == "development" ]]; then \ + sed -i 's/appVersion: "0.0.0-replaceme"/appVersion: '$(COMPONENT_DEV_VERSION)'/' ${HELM_TARGET_DIR}/Chart.yaml; \ + sed -i 's/version: 0.0.0-replaceme/version: '$(COMPONENT_DEV_VERSION)'/' ${HELM_TARGET_DIR}/Chart.yaml; \ + else \ + sed -i 's/appVersion: "0.0.0-replaceme"/appVersion: "${VERSION}"/' ${HELM_TARGET_DIR}/Chart.yaml; \ + sed -i 's/version: 0.0.0-replaceme/version: ${VERSION}/' ${HELM_TARGET_DIR}/Chart.yaml; \ + fi + +.PHONY: copy-helm-files +copy-helm-files: + @echo "Copying Helm files..." + @rm -drf ${HELM_TARGET_DIR} # delete folder, so the chart is newly created. + @mkdir -p ${HELM_TARGET_DIR}/templates + @cp -r ${HELM_SOURCE_DIR}/** ${HELM_TARGET_DIR} + +.PHONY: validate-chart +validate-chart: + @if [ ! -f ${HELM_SOURCE_DIR}/Chart.yaml ] ; then \ + echo "Could not find source Helm chart under \$${HELM_SOURCE_DIR}/Chart.yaml" ; \ + exit 22 ; \ + fi + +.PHONY: helm-update-dependencies +helm-update-dependencies: ${BINARY_HELM} ## Update Helm chart dependencies + @$(BINARY_HELM) dependency update "${HELM_SOURCE_DIR}" + +##@ K8s - Helm dev targets + +.PHONY: helm-apply +helm-apply: ${BINARY_HELM} check-k8s-namespace-env-var ${IMAGE_IMPORT_TARGET} helm-generate ${HELM_PRE_APPLY_TARGETS} ## Generates and installs the Helm chart. + @echo "Apply generated helm chart" + @${BINARY_HELM} --kube-context="${KUBE_CONTEXT_NAME}" upgrade -i ${COMPONENT_ARTIFACT_ID} ${HELM_TARGET_DIR} ${BINARY_HELM_ADDITIONAL_UPGR_ARGS} --namespace ${NAMESPACE} + +.PHONY: helm-delete +helm-delete: ${BINARY_HELM} check-k8s-namespace-env-var ## Uninstalls the current Helm chart. + @echo "Uninstall helm chart" + @${BINARY_HELM} --kube-context="${KUBE_CONTEXT_NAME}" uninstall ${COMPONENT_ARTIFACT_ID} --namespace=${NAMESPACE} ${BINARY_HELM_ADDITIONAL_UNINST_ARGS} || true + +.PHONY: helm-reinstall +helm-reinstall: helm-delete helm-apply ## Uninstalls the current helm chart and reinstalls it. + +.PHONY: helm-chart-import +helm-chart-import: ${CHECK_VAR_TARGETS} helm-generate helm-package ${IMAGE_IMPORT_TARGET} ## Imports the currently available chart into the cluster-local registry. + @if [[ ${STAGE} == "development" ]]; then \ + echo "Import ${HELM_DEV_RELEASE_TGZ} into K8s cluster ${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE}..."; \ + ${BINARY_HELM} push ${HELM_DEV_RELEASE_TGZ} oci://${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ + else \ + echo "Import ${HELM_RELEASE_TGZ} into K8s cluster ${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE}..."; \ + ${BINARY_HELM} push ${HELM_RELEASE_TGZ} oci://${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ + fi + @echo "Done." + +##@ K8s - Helm release targets + +.PHONY: helm-generate-release +helm-generate-release: update-urls ## Generates the final helm chart with release URLs. + + +.PHONY: helm-package +helm-package: helm-delete-existing-tgz ${HELM_RELEASE_TGZ} ## Generates and packages the helm chart with release URLs. + +${HELM_RELEASE_TGZ}: ${BINARY_HELM} ${HELM_TARGET_DIR}/Chart.yaml ${HELM_POST_GENERATE_TARGETS} ## Generates and packages the helm chart with release URLs. + @echo "Package generated helm chart" + @if [[ ${STAGE} == "development" ]]; then \ + echo "WARNING: You are using a development environment" ; \ + fi + @${BINARY_HELM} package ${HELM_TARGET_DIR} -d ${HELM_TARGET_DIR} ${BINARY_HELM_ADDITIONAL_PACK_ARGS} + +.PHONY: helm-delete-existing-tgz +helm-delete-existing-tgz: ## Remove an existing Helm package from the target directory. + @echo "Delete ${HELM_RELEASE_TGZ}*" + @rm -f ${HELM_TARGET_DIR}/${COMPONENT_ARTIFACT_ID}-*.tgz + +##@ K8s - Helm lint targets + +.PHONY: helm-lint +helm-lint: $(BINARY_HELM) helm-generate + @$(BINARY_HELM) lint "${HELM_TARGET_DIR}" + +##@ K8s - Component dev targets + +.PHONY: component-generate +component-generate: ${K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML} ${COMPONENT_POST_GENERATE_TARGETS} ## Generate the component yaml resource. + +${K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML}: ${K8S_RESOURCE_TEMP_FOLDER} + @echo "Generating temporary K8s component resource: ${K8S_RESOURCE_COMPONENT}" + @if [[ ${STAGE} == "development" ]]; then \ + sed "s|NAMESPACE|$(HELM_ARTIFACT_NAMESPACE)|g" "${K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML}" | sed "s|NAME|$(COMPONENT_ARTIFACT_ID)|g" | sed "s|VERSION|$(COMPONENT_DEV_VERSION)|g" > "${K8S_RESOURCE_COMPONENT}"; \ + else \ + sed "s|NAMESPACE|$(HELM_ARTIFACT_NAMESPACE)|g" "${K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML}" | sed "s|NAME|$(COMPONENT_ARTIFACT_ID)|g" | sed "s|VERSION|$(VERSION)|g" > "${K8S_RESOURCE_COMPONENT}"; \ + fi + +.PHONY: component-apply +component-apply: isProduction check-k8s-namespace-env-var ${COMPONENT_PRE_APPLY_TARGETS} ${IMAGE_IMPORT_TARGET} helm-generate helm-chart-import component-generate ## Applies the component yaml resource to the actual defined context. + @kubectl apply -f "${K8S_RESOURCE_COMPONENT}" --namespace="${NAMESPACE}" --context="${KUBE_CONTEXT_NAME}" + @echo "Done." + +.PHONY: component-delete +component-delete: check-k8s-namespace-env-var component-generate $(K8S_POST_GENERATE_TARGETS) ## Deletes the component yaml resource from the actual defined context. + @kubectl delete -f "${K8S_RESOURCE_COMPONENT}" --namespace="${NAMESPACE}" --context="${KUBE_CONTEXT_NAME}" || true + @echo "Done." + +.PHONY: component-reinstall +component-reinstall: component-delete component-apply ## Reinstalls the component yaml resource from the actual defined context. diff --git a/build/make/k8s-component.tpl b/build/make/k8s-component.tpl new file mode 100644 index 0000000..fa0eaa6 --- /dev/null +++ b/build/make/k8s-component.tpl @@ -0,0 +1,13 @@ +# Use the property .spec.deployNamespace to define the namespace the component should be deployed to. +# Make environment variable 'COMPONENT_DEPLOY_NAMESPACE' is responsible for that. +# If 'COMPONENT_DEPLOY_NAMESPACE' is empty the property 'deployNamespace' will be deleted. +apiVersion: k8s.cloudogu.com/v1 +kind: Component +metadata: + name: NAME + labels: + app: ces +spec: + name: NAME + namespace: NAMESPACE + version: VERSION \ No newline at end of file diff --git a/build/make/k8s-controller.mk b/build/make/k8s-controller.mk new file mode 100644 index 0000000..ea3d457 --- /dev/null +++ b/build/make/k8s-controller.mk @@ -0,0 +1,56 @@ +# This script requires the k8s.mk script +include ${BUILD_DIR}/make/k8s-component.mk +include ${BUILD_DIR}/make/k8s-crd.mk + +## Variables + +# make sure to create a statically linked binary otherwise it may quit with +# "exec user process caused: no such file or directory" +GO_BUILD_FLAGS=-mod=vendor -a -tags netgo,osusergo $(LDFLAGS) -o $(BINARY) + +# remove DWARF symbol table and strip other symbols to shave ~13 MB from binary +ADDITIONAL_LDFLAGS=-extldflags -static -w -s + +# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +ENVTEST_K8S_VERSION = 1.23 +K8S_INTEGRATION_TEST_DIR=${TARGET_DIR}/k8s-integration-test + +##@ K8s - EcoSystem + +.PHONY: build +build: helm-apply ## Builds a new version of the dogu and deploys it into the K8s-EcoSystem. + +##@ Release + +.PHONY: controller-release +controller-release: ## Interactively starts the release workflow. + @echo "Starting git flow release..." + @build/make/release.sh controller-tool + +##@ K8s - Development + +.PHONY: build-controller +build-controller: ${SRC} compile ## Builds the controller Go binary. + +# Allows to perform tasks before locally running the controller +K8S_RUN_PRE_TARGETS ?= +.PHONY: run +run: generate-deepcopy $(K8S_RUN_PRE_TARGETS) ## Run a controller from your host. + go run -ldflags "-X main.Version=$(VERSION)" ./main.go + +##@ K8s - Integration test with envtest + +$(K8S_INTEGRATION_TEST_DIR): + @mkdir -p $@ + +.PHONY: k8s-integration-test +k8s-integration-test: $(K8S_INTEGRATION_TEST_DIR) ${ENVTEST} ## Run k8s integration tests. + @echo "Running K8s integration tests..." + @KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -tags=k8s_integration ./... -coverprofile ${K8S_INTEGRATION_TEST_DIR}/report-k8s-integration.out + +##@ Controller specific targets + +.PHONY: generate-deepcopy +generate-deepcopy: ${CONTROLLER_GEN} ## Generate code containing DeepCopy* method implementations. + @echo "Auto-generate deepcopy functions..." + @$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." diff --git a/build/make/k8s-crd.mk b/build/make/k8s-crd.mk new file mode 100644 index 0000000..aca625d --- /dev/null +++ b/build/make/k8s-crd.mk @@ -0,0 +1,122 @@ +# we set this default to maintain compatibility with CRDs that are still inside monorepos +APPEND_CRD_SUFFIX ?= true +ifeq ($(APPEND_CRD_SUFFIX), true) + ARTIFACT_CRD_ID = $(ARTIFACT_ID)-crd +else ifeq ($(APPEND_CRD_SUFFIX), false) + ARTIFACT_CRD_ID = $(ARTIFACT_ID) +endif +CRD_BUILD_VERSION := $(shell date +%s).$(TIMESTAMP) +DEV_CRD_VERSION ?= ${VERSION}-dev.${COMPONENT_BUILD_VERSION} +HELM_CRD_SOURCE_DIR ?= ${WORKDIR}/k8s/helm-crd +HELM_CRD_TARGET_DIR ?= $(K8S_RESOURCE_TEMP_FOLDER)/helm-crd +HELM_CRD_RELEASE_TGZ = ${HELM_CRD_TARGET_DIR}/${ARTIFACT_CRD_ID}-${VERSION}.tgz +HELM_CRD_DEV_RELEASE_TGZ = ${HELM_CRD_TARGET_DIR}/${ARTIFACT_CRD_ID}-${DEV_CRD_VERSION}.tgz + +K8S_RESOURCE_CRD_COMPONENT ?= "${K8S_RESOURCE_TEMP_FOLDER}/component-${ARTIFACT_CRD_ID}-${VERSION}.yaml" +K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML ?= $(BUILD_DIR)/make/k8s-component.tpl +# CRD_POST_MANIFEST_TARGETS can be used to post-process CRD YAMLs after their creation. +CRD_POST_MANIFEST_TARGETS ?= crd-add-labels + +# This can be used by external components to prevent generate and copy controller manifests by overriding with an empty value. +CRD_HELM_MANIFEST_TARGET?=manifests + +##@ K8s - CRD targets + +.PHONY: manifests +manifests: ${CONTROLLER_GEN} manifests-run ${CRD_POST_MANIFEST_TARGETS} ## Generate CustomResourceDefinition YAMLs. + +.PHONY: manifests-run +manifests-run: + @echo "Generate manifests..." + @$(CONTROLLER_GEN) crd paths="./..." output:crd:artifacts:config=${HELM_CRD_SOURCE_DIR}/templates + +.PHONY: crd-add-labels +crd-add-labels: $(BINARY_YQ) + @echo "Adding labels to CRD..." + @for file in ${HELM_CRD_SOURCE_DIR}/templates/*.yaml ; do \ + $(BINARY_YQ) -i e ".metadata.labels.app = \"ces\"" $${file} ;\ + $(BINARY_YQ) -i e ".metadata.labels.\"app.kubernetes.io/name\" = \"${ARTIFACT_CRD_ID}\"" $${file} ;\ + done + +.PHONY: crd-helm-generate ## Generates the Helm CRD chart +crd-helm-generate: ${CRD_HELM_MANIFEST_TARGET} validate-crd-chart ${HELM_CRD_TARGET_DIR}/Chart.yaml ${K8S_POST_CRD_HELM_GENERATE_TARGETS} + +# this is phony because of it is easier this way than the makefile-single-run way +.PHONY: ${HELM_CRD_TARGET_DIR}/Chart.yaml +${HELM_CRD_TARGET_DIR}/Chart.yaml: ${K8S_RESOURCE_TEMP_FOLDER} + @echo "Copying Helm CRD files..." + @rm -drf ${HELM_CRD_TARGET_DIR}/templates + @mkdir -p ${HELM_CRD_TARGET_DIR}/templates + @cp -r ${HELM_CRD_SOURCE_DIR}/** ${HELM_CRD_TARGET_DIR} + + @echo "Generate Helm CRD chart..." + @sed -i 's/name: artifact-crd-replaceme/name: ${ARTIFACT_CRD_ID}/' ${HELM_CRD_TARGET_DIR}/Chart.yaml + @if [[ ${STAGE} == "development" ]]; then \ + sed -i 's/appVersion: "0.0.0-replaceme"/appVersion: "${DEV_CRD_VERSION}"/' ${HELM_CRD_TARGET_DIR}/Chart.yaml; \ + sed -i 's/version: 0.0.0-replaceme/version: ${DEV_CRD_VERSION}/' ${HELM_CRD_TARGET_DIR}/Chart.yaml; \ + else \ + sed -i 's/appVersion: "0.0.0-replaceme"/appVersion: "${VERSION}"/' ${HELM_CRD_TARGET_DIR}/Chart.yaml; \ + sed -i 's/version: 0.0.0-replaceme/version: ${VERSION}/' ${HELM_CRD_TARGET_DIR}/Chart.yaml; \ + fi + +.PHONY: validate-crd-chart +validate-crd-chart: + @if [ ! -f ${HELM_CRD_SOURCE_DIR}/Chart.yaml ] ; then \ + echo "Could not find CRD source Helm chart under \$${HELM_CRD_SOURCE_DIR}/Chart.yaml" ; \ + exit 23 ; \ + fi + +.PHONY: crd-helm-apply +crd-helm-apply: ${BINARY_HELM} check-k8s-namespace-env-var crd-helm-generate ## Generates and installs the Helm CRD chart. + @echo "Apply generated Helm CRD chart" + @${BINARY_HELM} --kube-context="${KUBE_CONTEXT_NAME}" upgrade -i ${ARTIFACT_CRD_ID} ${HELM_CRD_TARGET_DIR} ${BINARY_HELM_ADDITIONAL_UPGR_ARGS} --namespace ${NAMESPACE} + +.PHONY: crd-helm-delete +crd-helm-delete: ${BINARY_HELM} check-k8s-namespace-env-var ## Uninstalls the current Helm CRD chart. + @echo "Uninstall Helm CRD chart" + @${BINARY_HELM} --kube-context="${KUBE_CONTEXT_NAME}" uninstall ${ARTIFACT_CRD_ID} --namespace=${NAMESPACE} ${BINARY_HELM_ADDITIONAL_UNINST_ARGS} || true + +.PHONY: crd-helm-package +crd-helm-package: crd-helm-delete-existing-tgz ${HELM_CRD_RELEASE_TGZ} ## Generates and packages the Helm CRD chart. + +.PHONY: crd-helm-delete-existing-tgz +crd-helm-delete-existing-tgz: ## Remove an existing Helm CRD package. + @rm -f ${HELM_CRD_TARGET_DIR}/${ARTIFACT_CRD_ID}-*.tgz + +${HELM_CRD_RELEASE_TGZ}: ${BINARY_HELM} crd-helm-generate ## Generates and packages the Helm CRD chart. + @echo "Package generated helm crd-chart" + @${BINARY_HELM} package ${HELM_CRD_TARGET_DIR} -d ${HELM_CRD_TARGET_DIR} ${BINARY_HELM_ADDITIONAL_PACK_ARGS} + +.PHONY: crd-helm-chart-import +crd-helm-chart-import: ${CHECK_VAR_TARGETS} check-k8s-artifact-id crd-helm-generate crd-helm-package ## Imports the currently available Helm CRD chart into the cluster-local registry. + @if [[ ${STAGE} == "development" ]]; then \ + echo "Import ${HELM_CRD_DEV_RELEASE_TGZ} into K8s cluster ${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE}..."; \ + ${BINARY_HELM} push ${HELM_CRD_DEV_RELEASE_TGZ} oci://${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ + else \ + echo "Import ${HELM_CRD_RELEASE_TGZ} into K8s cluster ${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE}..."; \ + ${BINARY_HELM} push ${HELM_CRD_RELEASE_TGZ} oci://${CES_REGISTRY_HOST}/${HELM_ARTIFACT_NAMESPACE} ${BINARY_HELM_ADDITIONAL_PUSH_ARGS}; \ + fi + @echo "Done." + +.PHONY: crd-helm-lint +crd-helm-lint: $(BINARY_HELM) crd-helm-generate + @$(BINARY_HELM) lint "${HELM_CRD_TARGET_DIR}" + +.PHONY: crd-component-generate +crd-component-generate: ${K8S_RESOURCE_TEMP_FOLDER} ## Generate the CRD component YAML resource. + @echo "Generating temporary K8s crd-component resource: ${K8S_RESOURCE_CRD_COMPONENT}" + @if [[ ${STAGE} == "development" ]]; then \ + sed "s|NAMESPACE|$(HELM_ARTIFACT_NAMESPACE)|g" "${K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML}" | sed "s|NAME|$(ARTIFACT_CRD_ID)|g" | sed "s|VERSION|$(DEV_CRD_VERSION)|g" > "${K8S_RESOURCE_CRD_COMPONENT}"; \ + else \ + sed "s|NAMESPACE|$(HELM_ARTIFACT_NAMESPACE)|g" "${K8S_RESOURCE_COMPONENT_CR_TEMPLATE_YAML}" | sed "s|NAME|$(ARTIFACT_CRD_ID)|g" | sed "s|VERSION|$(VERSION)|g" > "${K8S_RESOURCE_CRD_COMPONENT}"; \ + fi + +.PHONY: crd-component-apply +crd-component-apply: isProduction check-k8s-namespace-env-var crd-helm-chart-import crd-component-generate ## Applies the CRD component YAML resource to the actual defined context. + @kubectl apply -f "${K8S_RESOURCE_CRD_COMPONENT}" --namespace="${NAMESPACE}" --context="${KUBE_CONTEXT_NAME}" + @echo "Done." + +.PHONY: crd-component-delete +crd-component-delete: check-k8s-namespace-env-var crd-component-generate ## Deletes the CRD component YAML resource from the actual defined context. + @kubectl delete -f "${K8S_RESOURCE_CRD_COMPONENT}" --namespace="${NAMESPACE}" --context="${KUBE_CONTEXT_NAME}" || true + @echo "Done." diff --git a/build/make/k8s-dogu.mk b/build/make/k8s-dogu.mk new file mode 100644 index 0000000..30e7a03 --- /dev/null +++ b/build/make/k8s-dogu.mk @@ -0,0 +1,51 @@ +# Variables +# Path to the dogu json of the dogu +DOGU_JSON_FILE=${WORKDIR}/dogu.json +DOGU_JSON_DEV_FILE=${WORKDIR}/${TARGET_DIR}/dogu.json +# Name of the dogu is extracted from the dogu.json +ARTIFACT_ID=$(shell $(BINARY_YQ) -oy -e ".Name" $(DOGU_JSON_FILE) | sed "s|.*/||g") +# Namespace of the dogu is extracted from the dogu.json +ARTIFACT_NAMESPACE=$(shell $(BINARY_YQ) -oy -e ".Name" $(DOGU_JSON_FILE) | sed "s|/.*||g") +# Version of the dogu is extracted from the dogu.json +VERSION=$(shell $(BINARY_YQ) -oy -e ".Version" $(DOGU_JSON_FILE)) +# Image of the dogu is extracted from the dogu.json +IMAGE=$(shell $(BINARY_YQ) -oy -e ".Image" $(DOGU_JSON_FILE)):$(VERSION) + +PRE_BUILD_TARGETS ?= + +ifeq (${K8S_MK_INCLUDE_MARKER}, ) + include ${BUILD_DIR}/make/k8s.mk +endif + +##@ K8s - EcoSystem + +.PHONY: build +build: ${PRE_BUILD_TARGETS} image-import install-dogu-descriptor create-dogu-resource apply-dogu-resource ## Builds a new version of the dogu and deploys it into the K8s-EcoSystem. + +##@ K8s - Dogu - Resource + +# The additional k8s yaml files +K8S_RESOURCE_PRODUCTIVE_FOLDER ?= $(WORKDIR)/k8s +K8S_RESOURCE_PRODUCTIVE_YAML ?= $(K8S_RESOURCE_PRODUCTIVE_FOLDER)/$(ARTIFACT_ID).yaml +K8S_RESOURCE_DOGU_CR_TEMPLATE_YAML ?= $(BUILD_DIR)/make/k8s-dogu.tpl +K8S_RESOURCE_DOGU ?= $(K8S_RESOURCE_TEMP_FOLDER)/$(ARTIFACT_ID).yaml +# The pre generation script creates a k8s resource yaml containing the dogu crd and the content from the k8s folder. +.PHONY: create-dogu-resource +create-dogu-resource: ${BINARY_YQ} $(K8S_RESOURCE_TEMP_FOLDER) + @echo "Generating temporary K8s resources $(K8S_RESOURCE_DOGU)..." + @rm -f $(K8S_RESOURCE_DOGU) + @sed "s|NAMESPACE|$(ARTIFACT_NAMESPACE)|g" $(K8S_RESOURCE_DOGU_CR_TEMPLATE_YAML) | sed "s|NAME|$(ARTIFACT_ID)|g" | sed "s|VERSION|$(VERSION)|g" >> $(K8S_RESOURCE_DOGU) + @echo "Done." + +.PHONY: apply-dogu-resource +apply-dogu-resource: + @kubectl --context="${KUBE_CONTEXT_NAME}" --namespace=${NAMESPACE} apply -f "$(K8S_RESOURCE_DOGU)" + +##@ K8s - Dogu + +.PHONY: install-dogu-descriptor +install-dogu-descriptor: ${BINARY_YQ} $(TARGET_DIR) ## Installs a configmap with current dogu.json into the cluster. + @echo "Generate configmap from dogu.json..." + @$(BINARY_YQ) -oj ".Image=\"${IMAGE_DEV}\" | .Version=\"${VERSION}\"" ${DOGU_JSON_FILE} > ${DOGU_JSON_DEV_FILE} + @kubectl --context="${KUBE_CONTEXT_NAME}" create configmap "$(ARTIFACT_ID)-descriptor" --from-file=$(DOGU_JSON_DEV_FILE) --dry-run=client -o yaml | kubectl --context="${KUBE_CONTEXT_NAME}" --namespace=${NAMESPACE} apply -f - + @echo "Done." diff --git a/build/make/k8s-dogu.tpl b/build/make/k8s-dogu.tpl new file mode 100644 index 0000000..91e2bb2 --- /dev/null +++ b/build/make/k8s-dogu.tpl @@ -0,0 +1,9 @@ +apiVersion: k8s.cloudogu.com/v2 +kind: Dogu +metadata: + name: NAME + labels: + app: ces +spec: + name: NAMESPACE/NAME + version: VERSION \ No newline at end of file diff --git a/build/make/k8s.mk b/build/make/k8s.mk new file mode 100644 index 0000000..d8e44a5 --- /dev/null +++ b/build/make/k8s.mk @@ -0,0 +1,241 @@ +# This file is optional and can be used to set personal information without committing them to the repository. +MY_ENV_FILE ?= $(WORKDIR)/.env +ifneq (,$(wildcard $(MY_ENV_FILE))) + include .env +endif + +## Variables + +K8S_MK_INCLUDE_MARKER="k8s.mk" + +BINARY_YQ = $(UTILITY_BIN_PATH)/yq +BINARY_YQ_4_VERSION?=v4.40.3 + +BINARY_HELM = $(UTILITY_BIN_PATH)/helm +BINARY_HELM_VERSION?=v3.20.2 +BINARY_HELM_URL?=https://get.helm.sh/helm-${BINARY_HELM_VERSION}-linux-amd64.tar.gz +BINARY_HELM_SUM?=258e830a9e613c8a7a302d6059b4bb3b9758f2f3e1bb8ea0d707ce10a9a72fea +BINARY_HELM_ARCHIVE_PATH?=linux-amd64/helm +BINARY_HELM_ARCHIVE_STRIP?=1 + +CONTROLLER_GEN = $(UTILITY_BIN_PATH)/controller-gen +CONTROLLER_GEN_VERSION?=v0.19.0 + +BINARY_CRANE_VERSION=v0.21.4 +BINARY_CRANE=$(UTILITY_BIN_PATH)/crane +BINARY_CRANE_URL?=https://github.com/google/go-containerregistry/releases/download/${BINARY_CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz +BINARY_CRANE_SUM?=3b6032bcf412e14cf3baf964a4065f2966af906ec947ab22478df5f74705c892 +BINARY_CRANE_ARCHIVE_PATH?=crane +BINARY_CRANE_ARCHIVE_STRIP?=0 + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +# The productive tag of the image +IMAGE ?= + +# Set production as default stage. Use "development" as stage in your .env file to generate artifacts +# with development images pointing to CES_REGISTRY_URL_PREFIX. +STAGE?=production + +# Set the "local" as runtime-environment, to push images to the container-registry of the local cluster and to apply resources to the local cluster. +# Use "remote" as runtime-environment in your .env file to push images to the container-registry at "registry.cloudogu.com/testing" and to apply resources to the configured kubernetes-context in KUBE_CONTEXT_NAME. +RUNTIME_ENV?=local +$(info RUNTIME_ENV=$(RUNTIME_ENV)) + +# The host and port of the local cluster +K3S_CLUSTER_FQDN?=k3ces.localdomain +K3S_LOCAL_REGISTRY_PORT?=30099 + +# The URL of the container-registry to use. Defaults to the registry of the local-cluster. +# If RUNTIME_ENV is "remote" it is "registry.cloudogu.com/testing", if ENVIRONMENT is "ci" it is "registry.cloudogu.com/ci" +# if run on ci (jenkins) the images must be pushed to a separate namespace in order to free space every night after the build. +CES_REGISTRY_HOST?=${K3S_CLUSTER_FQDN}:${K3S_LOCAL_REGISTRY_PORT} +CES_REGISTRY_NAMESPACE ?= +ifeq (${RUNTIME_ENV}, remote) + CES_REGISTRY_HOST=registry.cloudogu.com + CES_REGISTRY_NAMESPACE=/testing + ifeq ($(ENVIRONMENT), ci) + CES_REGISTRY_NAMESPACE=/ci + endif +endif +$(info CES_REGISTRY_HOST=$(CES_REGISTRY_HOST)) + +# The name of the kube-context to use for applying resources. +# If KUBE_CONTEXT_NAME is empty and RUNTIME_ENV is "remote" the currently configured kube-context is used. +# If KUBE_CONTEXT_NAME is empty and RUNTIME_ENV is not "remote" the "k3ces.localdomain" is used as kube-context. +ifeq (${KUBE_CONTEXT_NAME}, ) + ifeq (${RUNTIME_ENV}, remote) + KUBE_CONTEXT_NAME = $(shell kubectl config current-context) + else + KUBE_CONTEXT_NAME = k3ces.localdomain + endif +endif +$(info KUBE_CONTEXT_NAME=$(KUBE_CONTEXT_NAME)) + +# The git branch-name in lowercase, shortened to 63 bytes, and with everything except 0-9 and a-z replaced with -. No leading / trailing -. +GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9]+/-/g; s/^-+|-+$$//g' | cut -c1-63) +# The short git commit-hash +GIT_HASH := $(shell git rev-parse --short HEAD) + +## Image URL to use all building/pushing image targets +IMAGE_DEV?=$(CES_REGISTRY_HOST)$(CES_REGISTRY_NAMESPACE)/$(ARTIFACT_ID)/$(GIT_BRANCH) +IMAGE_DEV_VERSION=$(IMAGE_DEV):$(VERSION) + +# Variables for the temporary yaml files. These are used as template to generate a development resource containing +# the current namespace and the dev image. +K8S_RESOURCE_TEMP_FOLDER ?= $(TARGET_DIR)/k8s + +# This can be used by components with own images to check if all image env var are set. +# These components should override this variable with `check-all-vars`. +CHECK_VAR_TARGETS?=check-all-vars-without-image + +##@ K8s - Variables + +.PHONY: check-all-vars +check-all-vars: check-all-vars-without-image check-all-image-vars ## Conduct a sanity check against selected build artefacts or local environment + +.PHONY: check-all-image-vars +check-all-image-vars: check-k8s-image-env-var check-k8s-image-dev-var check-etc-hosts check-insecure-cluster-registry + +.PHONY: check-all-vars-without-image +check-all-vars-without-image: check-k8s-artifact-id check-k8s-namespace-env-var + +.PHONY: check-k8s-namespace-env-var +check-k8s-namespace-env-var: + @$(call check_defined, NAMESPACE, k8s namespace) + +.PHONY: check-k8s-image-env-var +check-k8s-image-env-var: + @$(call check_defined, IMAGE, docker image tag) + +.PHONY: check-k8s-artifact-id +check-k8s-artifact-id: + @$(call check_defined, ARTIFACT_ID, app/dogu name) + +.PHONY: check-etc-hosts +check-etc-hosts: + @if [[ ${RUNTIME_ENV} == "local" ]]; then \ + grep -E "^.+\s+${K3S_CLUSTER_FQDN}\$$" /etc/hosts > /dev/null || \ + (echo "Missing /etc/hosts entry for ${K3S_CLUSTER_FQDN}" && exit 1) \ + fi + +.PHONY: check-insecure-cluster-registry +check-insecure-cluster-registry: + @if [[ ${RUNTIME_ENV} == "local" ]]; then \ + grep "${CES_REGISTRY_HOST}" /etc/docker/daemon.json > /dev/null || \ + (echo "Missing /etc/docker/daemon.json for ${CES_REGISTRY_HOST}" && exit 1) \ + fi + +# If the RUNTIME_ENV is "remote" checks if the current docker-client has credentials for CES_REGISTRY_HOST +# If no credentials could be found, the credentials are queried and docker-login is performed +check-docker-credentials: + @if [[ "$(RUNTIME_ENV)" == "remote" ]]; then \ + if ! grep -q $(CES_REGISTRY_HOST) ~/.docker/config.json ; then \ + echo "Error: Docker is not logged in to $(CES_REGISTRY_HOST)"; \ + read -p "Enter Docker Username for $(CES_REGISTRY_HOST): " username; \ + read -sp "Enter Docker Password for $(CES_REGISTRY_HOST): " password; \ + echo ""; \ + echo "$$password" | docker login -u "$$username" --password-stdin $(CES_REGISTRY_HOST); \ + if [ $$? -eq 0 ]; then \ + echo "Docker login to $(CES_REGISTRY_HOST) successful"; \ + else \ + echo "Docker login to $(CES_REGISTRY_HOST) failed"; \ + exit 1; \ + fi \ + fi \ + fi + +##@ K8s - Resources + +${K8S_RESOURCE_TEMP_FOLDER}: + @mkdir -p $@ + + +##@ K8s - Docker + +.PHONY: docker-build +docker-build: check-docker-credentials check-k8s-image-env-var ${BINARY_YQ} ## Builds the docker image of the K8s app. + @echo "Building docker image $(IMAGE)..." + @DOCKER_BUILDKIT=1 docker build . -t $(IMAGE) + +.PHONY: docker-dev-tag +docker-dev-tag: check-k8s-image-dev-var docker-build ## Tags a Docker image for local K3ces deployment. + @echo "Tagging image with dev tag $(IMAGE_DEV_VERSION)..." + @DOCKER_BUILDKIT=1 docker tag ${IMAGE} $(IMAGE_DEV_VERSION) + +.PHONY: check-k8s-image-dev-var +check-k8s-image-dev-var: +ifeq (${IMAGE_DEV},) + @echo "Missing make variable IMAGE_DEV detected. It should look like \$${CES_REGISTRY_HOST}/docker-image:tag" + @exit 19 +endif + +.PHONY: image-import +image-import: check-all-vars check-k8s-artifact-id docker-dev-tag ## Imports the currently available image into the configured ces-registry. + @echo "Import $(IMAGE_DEV_VERSION) into K8s cluster ${KUBE_CONTEXT_NAME}..." + @docker push $(IMAGE_DEV_VERSION) + @echo "Done." + +## Functions + +# Check that given variables are set and all have non-empty values, +# die with an error otherwise. +# +# Params: +# 1. Variable name(s) to test. +# 2. (optional) Error message to print. +check_defined = \ + $(strip $(foreach 1,$1, \ + $(call __check_defined,$1,$(strip $(value 2))))) +__check_defined = \ + $(if $(value $1),, \ + $(error Undefined $1$(if $2, ($2)))) + +##@ K8s - Download Utilities + +.PHONY: install-yq ## Installs the yq YAML editor. +install-yq: ${BINARY_YQ} + +${BINARY_YQ}: $(UTILITY_BIN_PATH) + $(call go-get-tool,$(BINARY_YQ),github.com/mikefarah/yq/v4@${BINARY_YQ_4_VERSION}) + +##@ K8s - Download Kubernetes Utilities + +.PHONY: install-helm ## Download helm locally if necessary. +install-helm: ${BINARY_HELM} + +${BINARY_HELM}: $(UTILITY_BIN_PATH) + $(call curl-get-tool-from-tar,$(BINARY_HELM),$(BINARY_HELM_URL),$(BINARY_HELM_SUM),$(BINARY_HELM_ARCHIVE_PATH),$(BINARY_HELM_ARCHIVE_STRIP)) + +.PHONY: install-crane ## Installs crane. +install-crane: ${BINARY_CRANE} + +${BINARY_CRANE}: $(UTILITY_BIN_PATH) + $(call curl-get-tool-from-tar,$(BINARY_CRANE),$(BINARY_CRANE_URL),$(BINARY_CRANE_SUM),$(BINARY_CRANE_ARCHIVE_PATH),$(BINARY_CRANE_ARCHIVE_STRIP)) + +.PHONY: controller-gen +controller-gen: ${CONTROLLER_GEN} ## Download controller-gen locally if necessary. + +${CONTROLLER_GEN}: + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@${CONTROLLER_GEN_VERSION}) + +ENVTEST = $(UTILITY_BIN_PATH)/setup-envtest +.PHONY: envtest +envtest: ${ENVTEST} ## Download envtest-setup locally if necessary. + +${ENVTEST}: + $(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest) + +.PHONY: isProduction +isProduction: + @if [[ "${STAGE}" == "production" ]]; then \ + echo "Command executed in production stage. Aborting."; \ + exit 1; \ + else \ + echo "Command executed in development stage. Continuing."; \ + fi + + diff --git a/build/make/mockery.yaml b/build/make/mockery.yaml new file mode 100644 index 0000000..67b4339 --- /dev/null +++ b/build/make/mockery.yaml @@ -0,0 +1,4 @@ +inpackage: True +testonly: True +with-expecter: True +keeptree: False \ No newline at end of file diff --git a/build/make/mocks.mk b/build/make/mocks.mk new file mode 100644 index 0000000..82723c3 --- /dev/null +++ b/build/make/mocks.mk @@ -0,0 +1,27 @@ +##@ Mocking + +MOCKERY_BIN=${UTILITY_BIN_PATH}/mockery +MOCKERY_VERSION?=v2.53.3 +MOCKERY_YAML=${WORKDIR}/.mockery.yaml + +${MOCKERY_BIN}: ${UTILITY_BIN_PATH} + $(call go-get-tool,$(MOCKERY_BIN),github.com/vektra/mockery/v2@$(MOCKERY_VERSION)) + +${MOCKERY_YAML}: + @cp ${BUILD_DIR}/make/mockery.yaml ${WORKDIR}/.mockery.yaml + +.PHONY: mocks +mocks: ${MOCKERY_BIN} ${MOCKERY_YAML} ## This target is used to generate mocks for all interfaces in a project. + @for dir in ${WORKDIR}/*/ ;\ + do \ + # removes trailing '/' \ + dir=$${dir%*/} ;\ + # removes everything before the last '/' \ + dir=$${dir##*/} ;\ + if ! echo '${MOCKERY_IGNORED}' | egrep -q "\b$${dir}\b" ;\ + then \ + echo "Creating mocks for $${dir}" ;\ + ${MOCKERY_BIN} --all --dir $${dir} ;\ + fi ;\ + done ; + @echo "Mocks successfully created." diff --git a/build/make/package-debian.mk b/build/make/package-debian.mk index 168e8e2..ad55ea1 100644 --- a/build/make/package-debian.mk +++ b/build/make/package-debian.mk @@ -1,3 +1,5 @@ +##@ Debian packaging + # This Makefile holds all targets for building a debian package # For deployment of the deb package include the deploy-debian.mk! @@ -8,10 +10,10 @@ CONFFILES_FILE_TMP="$(DEBIAN_CONTENT_DIR)/conffiles_" DEBSRC:=$(shell find "${WORKDIR}/deb" -type f) .PHONY: package -package: debian-with-binary +package: debian-with-binary ## Build binary and package into .deb file .PHONY: debian -debian: $(DEBIAN_PACKAGE) +debian: $(DEBIAN_PACKAGE) ## Create .deb package without building the binary before .PHONY: debian-with-binary debian-with-binary: $(BINARY) $(DEBIAN_PACKAGE) diff --git a/build/make/package-tar.mk b/build/make/package-tar.mk index 307212f..9b842a3 100644 --- a/build/make/package-tar.mk +++ b/build/make/package-tar.mk @@ -1,7 +1,9 @@ +##@ Tar packaging + TAR_PACKAGE:=$(ARTIFACT_ID)-$(VERSION).tar.gz .PHONY: package -package: $(TAR_PACKAGE) +package: $(TAR_PACKAGE) ## Build binary and create tar package from it $(TAR_PACKAGE): $(BINARY) # Check owner and group id diff --git a/build/make/prerelease.mk b/build/make/prerelease.mk new file mode 100644 index 0000000..5ffc3bb --- /dev/null +++ b/build/make/prerelease.mk @@ -0,0 +1,6 @@ +# used to create switch the dogu to a prerelease namespace +# e.g. official/usermgmt -> prerelease_official/usermgmt + +.PHONY: prerelease_namespace +prerelease_namespace: + build/make/prerelease.sh prerelease_namespace \ No newline at end of file diff --git a/build/make/prerelease.sh b/build/make/prerelease.sh new file mode 100755 index 0000000..ba68b73 --- /dev/null +++ b/build/make/prerelease.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +prerelease_namespace() { + + TIMESTAMP=$(date +"%Y%m%d%H%M%S") + + # Update version in dogu.json + if [ -f "dogu.json" ]; then + echo "Updating name in dogu.json..." + ORIG_NAME="$(jq -r ".Name" ./dogu.json)" + ORIG_VERSION="$(jq -r ".Version" ./dogu.json)" + PRERELEASE_NAME="prerelease_${ORIG_NAME}" + PRERELEASE_VERSION="${ORIG_VERSION}${TIMESTAMP}" + jq ".Name = \"${PRERELEASE_NAME}\"" dogu.json >dogu2.json && mv dogu2.json dogu.json + jq ".Version = \"${PRERELEASE_VERSION}\"" dogu.json >dogu2.json && mv dogu2.json dogu.json + jq ".Image = \"registry.cloudogu.com/${PRERELEASE_NAME}\"" dogu.json >dogu2.json && mv dogu2.json dogu.json + fi + + # Update version in Dockerfile + if [ -f "Dockerfile" ]; then + echo "Updating version in Dockerfile..." + LABEL_BLOCK=$(sed -n '/^LABEL[[:space:]]/ {N; /NAME=".*"/ {N; /VERSION=".*"/ {p}}}' Dockerfile) + + # Extract NAME and VERSION from the LABEL block + ORIG_NAME=$(echo "$LABEL_BLOCK" | sed -n 's/.*NAME="\([^"]*\)".*/\1/p') + ORIG_VERSION=$(echo "$LABEL_BLOCK" | sed -n 's/.*VERSION="\([^"]*\)".*/\1/p') + + # Output the extracted values for debugging + echo "ORIG_NAME Dockerfile: ${ORIG_NAME}" + echo "ORIG_VERSION Dockerfile: ${ORIG_VERSION}" + + # Prepare prerelease name and version + PRERELEASE_NAME="prerelease_$(echo -e "$ORIG_NAME" | sed 's/\//\\\//g')" + PRERELEASE_VERSION="${ORIG_VERSION}${TIMESTAMP}" + + # Output the new values for debugging + echo "PRERELEASE_NAME Dockerfile: ${PRERELEASE_NAME}" + echo "PRERELEASE_VERSION Dockerfile: ${PRERELEASE_VERSION}" + + # Only replace NAME= and VERSION= and only inside the LABEL block + # This assumes LABEL block is between 'LABEL' and first non-indented line + sed -i '/^LABEL/,/^[^[:space:]]/ { + s/\(NAME="\)[^"]*\("\)/\1'"${PRERELEASE_NAME}"'\2/ + s/\(VERSION="\)[^"]*\("\)/\1'"${PRERELEASE_VERSION}"'\2/ + }' Dockerfile + fi + +} + + +TYPE="${1}" + +echo ${TYPE} +if [[ "${TYPE}" == "prerelease_namespace" ]];then + prerelease_namespace +fi \ No newline at end of file diff --git a/build/make/release.mk b/build/make/release.mk index d8a5892..1fab1d6 100644 --- a/build/make/release.mk +++ b/build/make/release.mk @@ -1,6 +1,23 @@ +##@ Releases + # This makefile holds the dogu-release target for starting a new dogu release .PHONY: dogu-release -dogu-release: - build/make/release.sh +dogu-release: ## Start a dogu release + build/make/release.sh dogu "${FIXED_CVE_LIST}" $(DRY_RUN) + +.PHONY: node-release +node-release: ## Start a node package release + build/make/release.sh node-pkg + +.PHONY: go-release +go-release: ## Start a go tool release + build/make/release.sh go-tool + +.PHONY: image-release +image-release: ## Start a go tool release + build/make/release.sh image +.PHONY: dogu-cve-release +dogu-cve-release: ## Start a dogu release of a new build if the local build fixes critical CVEs + @bash -c "build/make/release_cve.sh \"${REGISTRY_USERNAME}\" \"${REGISTRY_PASSWORD}\" \"${TRIVY_IMAGE_SCAN_FLAGS}\" \"${DRY_RUN}\" \"${CVE_SEVERITY}\"" diff --git a/build/make/release.sh b/build/make/release.sh index 07cd650..ee70624 100755 --- a/build/make/release.sh +++ b/build/make/release.sh @@ -3,143 +3,68 @@ set -o errexit set -o nounset set -o pipefail -wait_for_ok(){ - printf "\n" - OK=false - while [[ ${OK} != "ok" ]] ; do - read -r -p "${1} (type 'ok'): " OK - done +# Extension points in release.sh: +# +# A custom release argument file will be sourced if found. The custom release arg file may implement one or more bash +# functions which either release.sh or release_functions.sh define. If such a custom release function is found the +# release script must define the argument list which the custom release function will receive during the release. + +sourceCustomReleaseArgs() { + RELEASE_ARGS_FILE="${1}" + + if [[ -f "${RELEASE_ARGS_FILE}" ]]; then + echo "Using custom release args file ${RELEASE_ARGS_FILE}" + + local sourceCustomReleaseExitCode=0 + # shellcheck disable=SC1090 + source "${RELEASE_ARGS_FILE}" || sourceCustomReleaseExitCode=$? + if [[ ${sourceCustomReleaseExitCode} -ne 0 ]]; then + echo "Error while sourcing custom release arg file ${sourceCustomReleaseExitCode}. Exiting." + exit 9 + fi + fi } -ask_yes_or_no(){ - local ANSWER="" - - while [ "${ANSWER}" != "y" ] && [ "${ANSWER}" != "n" ]; do - read -r -p "${1} (type 'y/n'): " ANSWER - done +PROJECT_DIR="$(pwd)" +RELEASE_ARGS_FILE="${PROJECT_DIR}/release_args.sh" - echo "${ANSWER}" -} +sourceCustomReleaseArgs "${RELEASE_ARGS_FILE}" -# dogu.json will always exist. So get current dogu version from dogu.json. -CURRENT_DOGU_VERSION=$(jq ".Version" --raw-output dogu.json) +# shellcheck disable=SC1090 +source "$(pwd)/build/make/release_functions.sh" -# Enter the target version -read -r -p "Current Version is v${CURRENT_DOGU_VERSION}. Please provide the new version: v" NEW_RELEASE_VERSION +TYPE="${1}" +FIXED_CVE_LIST="${2:-""}" +DRY_RUN="${3:-""}" -# Validate that release version does not start with vv -if [[ ${NEW_RELEASE_VERSION} = v* ]]; then - echo "WARNING: The new release version (v${NEW_RELEASE_VERSION}) starts with 'vv'." - echo "You must not enter the v when defining the new version." - ANSWER=$(ask_yes_or_no "Should the first v be removed?") - if [ "${ANSWER}" == "y" ]; then - NEW_RELEASE_VERSION="${NEW_RELEASE_VERSION:1}" - echo "Release version now is: ${NEW_RELEASE_VERSION}" - fi -fi; +echo "=====Starting Release process=====" -# Do gitflow -git flow init --defaults --force - -mainBranchExists="$(git show-ref refs/remotes/origin/main)" -if [ -n "$mainBranchExists" ]; then - echo 'Using "main" branch for production releases' - git flow config set master main - git checkout main - git pull origin main +if [[ "${TYPE}" == "dogu" || "${TYPE}" == "dogu-cve-release" ]];then + CURRENT_TOOL_VERSION=$(get_current_version_by_dogu_json) else - echo 'Using "master" branch for production releases' - git checkout master - git pull origin master + CURRENT_TOOL_VERSION=$(get_current_version_by_makefile) fi -git checkout develop -git pull origin develop -git flow release start v"${NEW_RELEASE_VERSION}" - -# Update version in dogu.json -jq ".Version = \"${NEW_RELEASE_VERSION}\"" dogu.json > dogu2.json && mv dogu2.json dogu.json -# Update version in Dockerfile -sed -i "s/\(^[ ]*VERSION=\"\)\([^\"]*\)\(.*$\)/\1${NEW_RELEASE_VERSION}\3/" Dockerfile -# Update version in Makefile -if [ -f "Makefile" ]; then - sed -i "s/\(^VERSION=\)\(.*\)$/\1${NEW_RELEASE_VERSION}/" Makefile -fi -# Update version in package.json -if [ -f "package.json" ]; then - jq ".version = \"${NEW_RELEASE_VERSION}\"" package.json > package2.json && mv package2.json package.json -fi -# Update version in pom.xml -if [ -f "pom.xml" ]; then - echo "Updating version in pom.xml..." - mvn versions:set -DgenerateBackupPoms=false -DnewVersion="${NEW_RELEASE_VERSION}" -fi +BASE_VERSION=$(get_base_version_by_makefile) +NEW_RELEASE_VERSION="$(read_new_version)" -# Commit changes to version -wait_for_ok "Please make sure that all versions have been updated correctly now (e.g. via \"git diff\")." -git add Dockerfile -git add dogu.json -if [ -f "Makefile" ]; then - git add Makefile -fi -if [ -f "package.json" ]; then - git add package.json -fi -if [ -f "pom.xml" ]; then - git add pom.xml +validate_new_version "${NEW_RELEASE_VERSION}" +if [[ -n "${DRY_RUN}" ]]; then + start_dry_run_release "${NEW_RELEASE_VERSION}" +else + start_git_flow_release "${NEW_RELEASE_VERSION}" "${BASE_VERSION}" fi -git commit -m "Bump version" - -# Changelog update -CURRENT_DATE=$(date --rfc-3339=date) -NEW_CHANGELOG_TITLE="## [v${NEW_RELEASE_VERSION}] - ${CURRENT_DATE}" -# Check if "Unreleased" tag exists -while ! grep --silent "## \[Unreleased\]" CHANGELOG.md; do - echo "" - echo -e "\e[31mYour CHANGELOG.md does not contain a \"## [Unreleased]\" line!\e[0m" - echo "Please add one to make it comply to https://keepachangelog.com/en/1.0.0/" - wait_for_ok "Please insert a \"## [Unreleased]\" line into CHANGELOG.md now." -done - -# Add new title line to changelog -sed -i "s|## \[Unreleased\]|## \[Unreleased\]\n\n${NEW_CHANGELOG_TITLE}|g" CHANGELOG.md - -# Wait for user to validate changelog changes -wait_for_ok "Please make sure your CHANGELOG.md looks as desired." - -# Check if new version tag still exists -while ! grep --silent "## \[v${NEW_RELEASE_VERSION}\] - ${CURRENT_DATE}" CHANGELOG.md; do - echo "" - echo -e "\e[31mYour CHANGELOG.md does not contain \"${NEW_CHANGELOG_TITLE}\"!\e[0m" - wait_for_ok "Please update your CHANGELOG.md now." -done - -git add CHANGELOG.md -git commit -m "Update changelog" - -if ! git diff --exit-code > /dev/null; then - echo "There are still uncommitted changes:" - echo "" - echo "# # # # # # # # # #" - echo "" - git --no-pager diff - echo "" - echo "# # # # # # # # # #" + +update_versions "${NEW_RELEASE_VERSION}" +update_changelog "${NEW_RELEASE_VERSION}" "${FIXED_CVE_LIST}" +update_releasenotes "${NEW_RELEASE_VERSION}" +show_diff + +if [[ -n "${DRY_RUN}" ]]; then + abort_dry_run_release "${NEW_RELEASE_VERSION}" "${BASE_VERSION}" +else + finish_release_and_push "${CURRENT_TOOL_VERSION}" "${NEW_RELEASE_VERSION}" "${BASE_VERSION}" fi -echo "All changes compared to develop branch:" -echo "" -echo "# # # # # # # # # #" -echo "" -git --no-pager diff develop -echo "" -echo "# # # # # # # # # #" - -# Push changes and delete release branch -wait_for_ok "Dogu upgrade from version v${CURRENT_DOGU_VERSION} to version v${NEW_RELEASE_VERSION} finished. Should the changes be pushed?" -git push origin release/v"${NEW_RELEASE_VERSION}" - -echo "Switching back to develop and deleting branch release/v${NEW_RELEASE_VERSION}..." -git checkout develop -git branch -D release/v"${NEW_RELEASE_VERSION}" +echo "=====Finished Release process=====" diff --git a/build/make/release_cve.sh b/build/make/release_cve.sh new file mode 100755 index 0000000..6c5fc6d --- /dev/null +++ b/build/make/release_cve.sh @@ -0,0 +1,166 @@ +#!/bin/bash +set -o errexit +set -o pipefail +set -o nounset + +function readCveSeverityIfUnset() { + if [ -z "${CVE_SEVERITY}" ]; then + echo "CVE_SEVERITY is unset" + while [[ -z ${CVE_SEVERITY} ]]; do + read -r -p "select the desired cve severity (CRITICAL, HIGH, MEDIUM, ...): " CVE_SEVERITY + done + fi +} + +function readCredentialsIfUnset() { + if [ -z "${USERNAME}" ]; then + echo "username is unset" + while [[ -z ${USERNAME} ]]; do + read -r -p "type username for ${REGISTRY_URL}: " USERNAME + done + fi + if [ -z "${PASSWORD}" ]; then + echo "password is unset" + while [[ -z ${PASSWORD} ]]; do + read -r -s -p "type password for ${REGISTRY_URL}: " PASSWORD + done + fi +} + +function diffArrays() { + local cveListX=("$1") + local cveListY=("$2") + local result=() + + local cveX + # Disable the following shellcheck because the arrays are sufficiently whitespace delimited because of the jq parsing result. + # shellcheck disable=SC2128 + for cveX in ${cveListX}; do + local found=0 + local cveY + for cveY in ${cveListY}; do + [[ "${cveY}" == "${cveX}" ]] && { + found=1 + break + } + done + + [[ "${found}" == 0 ]] && result+=("${cveX}") + done + + echo "${result[@]}" +} + +function dockerLogin() { + docker login "${REGISTRY_URL}" -u "${USERNAME}" -p "${PASSWORD}" +} + +function dockerLogout() { + docker logout "${REGISTRY_URL}" +} + +function nameFromDogu() { + jsonPropertyFromDogu ".Name" +} + +function imageFromDogu() { + jsonPropertyFromDogu ".Image" +} + +function versionFromDogu() { + jsonPropertyFromDogu ".Version" +} + +function jsonPropertyFromDogu() { + local property="${1}" + jq -r "${property}" "${DOGU_JSON_FILE}" +} + +function pullRemoteImage() { + docker pull "$(imageFromDogu):$(versionFromDogu)" +} + +function buildLocalImage() { + docker build --no-cache . -t "$(imageFromDogu):$(versionFromDogu)" +} + +function scanImage() { + docker run -v "${TRIVY_CACHE_DIR}":"${TRIVY_DOCKER_CACHE_DIR}" -v /var/run/docker.sock:/var/run/docker.sock -v "${TRIVY_PATH}":/result aquasec/trivy --cache-dir "${TRIVY_DOCKER_CACHE_DIR}" -f json -o /result/results.json image ${TRIVY_IMAGE_SCAN_FLAGS:+"${TRIVY_IMAGE_SCAN_FLAGS}"} "$(imageFromDogu):$(versionFromDogu)" +} + +function parseTrivyJsonResult() { + local severity="${1}" + local trivy_result_file="${2}" + + # First select results which have the property "Vulnerabilities". Filter the vulnerability ids with the given severity and afterward put the values in an array. + # This array is used to format the values with join(" ") in a whitespace delimited string list. + jq -rc "[.Results[] | select(.Vulnerabilities) | .Vulnerabilities | .[] | select(.Severity == \"${severity}\") | .VulnerabilityID] | unique | join(\" \")" "${trivy_result_file}" +} + +RELEASE_SH="build/make/release.sh" + +REGISTRY_URL="registry.cloudogu.com" +DOGU_JSON_FILE="dogu.json" + +CVE_SEVERITY= + +TRIVY_PATH= +TRIVY_RESULT_FILE= +TRIVY_CACHE_DIR= +TRIVY_DOCKER_CACHE_DIR=/tmp/db +TRIVY_IMAGE_SCAN_FLAGS= + +USERNAME="" +PASSWORD="" +DRY_RUN= + +function runMain() { + readCveSeverityIfUnset + readCredentialsIfUnset + dockerLogin + + mkdir -p "${TRIVY_PATH}" # Cache will not be removed after release. rm requires root because the trivy container only runs with root. + pullRemoteImage + scanImage + local remote_trivy_cve_list + remote_trivy_cve_list=$(parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}") + + buildLocalImage + scanImage + local local_trivy_cve_list + local_trivy_cve_list=$(parseTrivyJsonResult "${CVE_SEVERITY}" "${TRIVY_RESULT_FILE}") + + dockerLogout + + local cve_in_local_but_not_in_remote + cve_in_local_but_not_in_remote=$(diffArrays "${local_trivy_cve_list}" "${remote_trivy_cve_list}") + if [[ -n "${cve_in_local_but_not_in_remote}" ]]; then + echo "Abort release. Added new vulnerabilities:" + echo "${cve_in_local_but_not_in_remote[@]}" + exit 2 + fi + + local cve_in_remote_but_not_in_local + cve_in_remote_but_not_in_local=$(diffArrays "${remote_trivy_cve_list}" "${local_trivy_cve_list}") + if [[ -z "${cve_in_remote_but_not_in_local}" ]]; then + echo "Abort release. Fixed no new vulnerabilities" + exit 3 + fi + + echo "Fixed ${CVE_SEVERITY} CVEs: ${cve_in_remote_but_not_in_local}" + "${RELEASE_SH}" "dogu-cve-release" "${cve_in_remote_but_not_in_local}" "${DRY_RUN}" +} + +# make the script only runMain when executed, not when sourced from bats tests +if [[ -n "${BASH_VERSION}" && "${BASH_SOURCE[0]}" == "${0}" ]]; then + USERNAME="${1:-""}" + PASSWORD="${2:-""}" + TRIVY_IMAGE_SCAN_FLAGS="${3:-""}" + DRY_RUN="${4:-""}" + CVE_SEVERITY="${5:-""}" + + TRIVY_PATH="/tmp/trivy-dogu-cve-release-$(nameFromDogu)" + TRIVY_RESULT_FILE="${TRIVY_PATH}/results.json" + TRIVY_CACHE_DIR="${TRIVY_PATH}/db" + runMain +fi diff --git a/build/make/release_functions.sh b/build/make/release_functions.sh new file mode 100755 index 0000000..0abe86e --- /dev/null +++ b/build/make/release_functions.sh @@ -0,0 +1,360 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +wait_for_ok() { + printf "\n" + local OK="false" + while [[ "${OK}" != "ok" ]]; do + read -r -p "${1} (type 'ok'): " OK + done +} + +ask_yes_or_no() { + local ANSWER="" + + while [ "${ANSWER}" != "y" ] && [ "${ANSWER}" != "n" ]; do + read -r -p "${1} (type 'y/n'): " ANSWER + done + + echo "${ANSWER}" +} + +get_current_version_by_makefile() { + grep '^VERSION=[0-9[:alpha:].-]*$' Makefile | sed s/VERSION=//g +} + +get_base_version_by_makefile() { + BASE_VERSION=$(grep '^BASE_VERSION=[0-9[:alpha:].-]*$' Makefile | sed s/BASE_VERSION=//g) + echo "${BASE_VERSION}" +} + +get_current_version_by_dogu_json() { + jq ".Version" --raw-output dogu.json +} + +read_new_version() { + local NEW_RELEASE_VERSION + read -r -p "Current Version is v${CURRENT_TOOL_VERSION}. Please provide the new version: v" NEW_RELEASE_VERSION + echo "${NEW_RELEASE_VERSION}" +} + +validate_new_version() { + local NEW_RELEASE_VERSION="${1}" + # Validate that release version does not start with vv + if [[ ${NEW_RELEASE_VERSION} = v* ]]; then + echo "WARNING: The new release version (v${NEW_RELEASE_VERSION}) starts with 'vv'." + echo "You must not enter the v when defining the new version." + local ANSWER + ANSWER=$(ask_yes_or_no "Should the first v be removed?") + if [ "${ANSWER}" == "y" ]; then + NEW_RELEASE_VERSION="${NEW_RELEASE_VERSION:1}" + echo "Release version now is: ${NEW_RELEASE_VERSION}" + fi + fi +} + +start_git_flow_release() { + local NEW_RELEASE_VERSION="${1}" + local BASE_RELEASE_VERSION="${2}" + local BASE_DEV_BRANCH_NAME + + # Do gitflow + git flow init --defaults --force + + local mainBranchExists + mainBranchExists="$(git show-ref refs/remotes/origin/main || echo "")" + if [[ -z "$BASE_RELEASE_VERSION" ]]; then + echo "BASE_RELEASE_VERSION variable is empty" + if [ -n "$mainBranchExists" ]; then + echo 'Using "main" branch for production releases' + git flow config set master main + git checkout main + git pull origin main + else + echo 'Using "master" branch for production releases' + git checkout master + git pull origin master + fi + BASE_DEV_BRANCH_NAME="develop" + else + echo "BASE_RELEASE_VERSION variable is not empty" + if [[ ${NEW_RELEASE_VERSION} != ${BASE_RELEASE_VERSION}* ]]; then + echo "ERROR: Release version (${NEW_RELEASE_VERSION}) does not start with base version (${BASE_RELEASE_VERSION})" + exit 1 + fi + + BASE_MAIN_BRANCH_NAME="${BASE_RELEASE_VERSION}/main" + echo "Using ${BASE_MAIN_BRANCH_NAME} branch for production releases" + git flow config set master ${BASE_MAIN_BRANCH_NAME} + git checkout ${BASE_MAIN_BRANCH_NAME} + git pull origin ${BASE_MAIN_BRANCH_NAME} + BASE_DEV_BRANCH_NAME="${BASE_RELEASE_VERSION}/develop" + fi + + git flow config set develop ${BASE_DEV_BRANCH_NAME} + + git checkout ${BASE_DEV_BRANCH_NAME} + git pull origin ${BASE_DEV_BRANCH_NAME} + git flow config + git flow release start v"${NEW_RELEASE_VERSION}" +} + +start_dry_run_release() { + local NEW_RELEASE_VERSION="${1}" + + git checkout -b dryrun/v"${NEW_RELEASE_VERSION}" +} + +abort_dry_run_release() { + local NEW_RELEASE_VERSION="${1}" + local BASE_RELEASE_VERSION="${2}" + + local BASE_DEV_BRANCH_NAME + + if [[ -z "$BASE_RELEASE_VERSION" ]]; then + BASE_DEV_BRANCH_NAME="develop" + else + BASE_DEV_BRANCH_NAME="${BASE_RELEASE_VERSION}/develop" + fi + + git checkout ${BASE_DEV_BRANCH_NAME} + git branch -D dryrun/v"${NEW_RELEASE_VERSION}" +} + +# update_versions updates files with the new release version and interactively asks the user for verification. If okay +# the updated files will be staged to git and finally committed. +# +# extension points: +# - update_versions_modify_files - update a file with the new version number +# - update_versions_stage_modified_files - stage a modified file to prepare the file for the up-coming commit +update_versions() { + local NEW_RELEASE_VERSION="${1}" + + if [[ $(type -t update_versions_modify_files) == function ]]; then + local preSkriptExitCode=0 + update_versions_modify_files "${NEW_RELEASE_VERSION}" || preSkriptExitCode=$? + if [[ ${preSkriptExitCode} -ne 0 ]]; then + echo "ERROR: custom update_versions_modify_files() exited with exit code ${preSkriptExitCode}" + exit 1 + fi + fi + + # Update version in dogu.json + if [ -f "dogu.json" ]; then + echo "Updating version in dogu.json..." + jq ".Version = \"${NEW_RELEASE_VERSION}\"" dogu.json >dogu2.json && mv dogu2.json dogu.json + fi + + # Update version in Dockerfile + if [ -f "Dockerfile" ]; then + echo "Updating version in Dockerfile..." + sed -i "s/\(^[ ]*VERSION=\"\)\([^\"]*\)\(.*$\)/\1${NEW_RELEASE_VERSION}\3/" Dockerfile + fi + + # Update version in Makefile + if [ -f "Makefile" ]; then + echo "Updating version in Makefile..." + sed -i "s/\(^VERSION=\)\(.*\)$/\1${NEW_RELEASE_VERSION}/" Makefile + fi + + # Update version in package.json + if [ -f "package.json" ]; then + echo "Updating version in package.json..." + jq ".version = \"${NEW_RELEASE_VERSION}\"" package.json >package2.json && mv package2.json package.json + fi + + # Update version in pom.xml + if [ -f "pom.xml" ]; then + echo "Updating version in pom.xml..." + mvn versions:set -DgenerateBackupPoms=false -DnewVersion="${NEW_RELEASE_VERSION}" + fi + + wait_for_ok "Please make sure that all versions have been updated correctly now (e.g. via \"git diff\")." + + ### The `git add` command has to be after the okay. Otherwise user-made changes to versions would not be added. + + if [[ $(type -t update_versions_stage_modified_files) == function ]]; then + preSkriptExitCode=0 + update_versions_stage_modified_files "${NEW_RELEASE_VERSION}" || preSkriptExitCode=$? + if [[ ${preSkriptExitCode} -ne 0 ]]; then + echo "ERROR: custom update_versions_stage_modified_files exited with exit code ${preSkriptExitCode}" + exit 1 + fi + fi + + if [ -f "dogu.json" ]; then + git add dogu.json + fi + + if [ -f "Dockerfile" ]; then + git add Dockerfile + fi + + if [ -f "Makefile" ]; then + git add Makefile + fi + + if [ -f "package.json" ]; then + git add package.json + fi + + if [ -f "pom.xml" ]; then + git add pom.xml + fi + + git commit -m "Bump version" +} + +update_changelog() { + local NEW_RELEASE_VERSION="${1}" + local FIXED_CVE_LIST="${2}" + + # Changelog update + local CURRENT_DATE + CURRENT_DATE=$(date --rfc-3339=date) + local NEW_CHANGELOG_TITLE="## [v${NEW_RELEASE_VERSION}] - ${CURRENT_DATE}" + # Check if "Unreleased" tag exists + while ! grep --silent "## \[Unreleased\]" CHANGELOG.md; do + echo "" + echo -e "\e[31mYour CHANGELOG.md does not contain a \"## [Unreleased]\" line!\e[0m" + echo "Please add one to make it comply to https://keepachangelog.com/en/1.0.0/" + wait_for_ok "Please insert a \"## [Unreleased]\" line into CHANGELOG.md now." + done + + if [[ -n "${FIXED_CVE_LIST}" ]]; then + addFixedCVEListFromReRelease "${FIXED_CVE_LIST}" + fi + + # Add new title line to changelog + sed -i "s|## \[Unreleased\]|## \[Unreleased\]\n\n${NEW_CHANGELOG_TITLE}|g" CHANGELOG.md + + # Wait for user to validate changelog changes + wait_for_ok "Please make sure your CHANGELOG.md looks as desired." + + # Check if new version tag still exists + while ! grep --silent "## \[v${NEW_RELEASE_VERSION}\] - ${CURRENT_DATE}" CHANGELOG.md; do + echo "" + echo -e "\e[31mYour CHANGELOG.md does not contain \"${NEW_CHANGELOG_TITLE}\"!\e[0m" + wait_for_ok "Please update your CHANGELOG.md now." + done + + git add CHANGELOG.md + git commit -m "Update changelog" +} + +update_releasenotes() { + local NEW_RELEASE_VERSION="${1}" + + # ReleaseNotes update + local CURRENT_DATE + CURRENT_DATE=$(date --rfc-3339=date) + local NEW_RELEASENOTE_TITLE="## [v${NEW_RELEASE_VERSION}] - ${CURRENT_DATE}" + rm -rf ".rn_changed" + find . -name "*release_notes*.md" -print0 | while read -d $'\0' file + do + # Check if "Unreleased" tag exists + while ! grep --silent "## \[Unreleased\]" "${file}"; do + echo "" + echo -e "\e[31mYour ${file} does not contain a \"## [Unreleased]\" line!\e[0m" + echo "Please add one to make it comply to https://keepachangelog.com/en/1.0.0/" + wait_for_ok "Please insert a \"## [Unreleased]\" line into ${file} now." + done + + # Add new title line to changelog + sed -i "s|## \[Unreleased\]|## \[Unreleased\]\n\n${NEW_RELEASENOTE_TITLE}|g" "${file}" + echo "Processed ${file}" + echo true > ".rn_changed" + done + + if test -f ".rn_changed" ; then + # Wait for user to validate changelog changes + wait_for_ok "Please make sure your release notes looks as desired." + + find . -name "*release_notes*.md" -print0 | while read -d $'\0' file + do + # Check if new version tag still exists + while ! grep --silent "$(echo $NEW_RELEASENOTE_TITLE | sed -e 's/[]\/$*.^[]/\\&/g')" "${file}"; do + echo "" + echo -e "\e[31mYour ${file} does not contain \"${NEW_RELEASENOTE_TITLE}\"!\e[0m" + wait_for_ok "Please update your ${file} now." + done + git add "${file}" + done + + git commit -m "Update ReleaseNotes" + fi + rm -rf ".rn_changed" +} + +# addFixedCVEListFromReRelease is used in dogu cve releases. The method adds the fixed CVEs under the ### Fixed header +# in the unreleased section. +addFixedCVEListFromReRelease() { + local fixed_cve_list="${1}" + + local cve_sed_search="" + local cve_sed_replace="" + local fixed_exists_in_unreleased + fixed_exists_in_unreleased=$(awk '/^\#\# \[Unreleased\]$/{flag=1;next}/^\#\# \[/{flag=0}flag' CHANGELOG.md | grep -e "^### Fixed$" || true) + if [[ -n "${fixed_exists_in_unreleased}" ]]; then + # extend fixed header with CVEs. + cve_sed_search="^\#\#\# Fixed$" + cve_sed_replace="\#\#\# Fixed\n- Fixed ${fixed_cve_list}" + else + # extend unreleased header with fixed header and CVEs. + cve_sed_search="^\#\# \[Unreleased\]$" + cve_sed_replace="\#\# \[Unreleased\]\n\#\#\# Fixed\n- Fixed ${fixed_cve_list}" + + local any_exists_unreleased + any_exists_unreleased=$(awk '/^\#\# \[Unreleased\]$/{flag=1;next}/^\#\# \[/{flag=0}flag' CHANGELOG.md | grep -e "^\#\#\# Added$" -e "^\#\#\# Fixed$" -e "^\#\#\# Changed$" || true) + if [[ -n ${any_exists_unreleased} ]]; then + cve_sed_replace+="\n" + fi + fi + + sed -i "0,/${cve_sed_search}/s//${cve_sed_replace}/" CHANGELOG.md +} + +show_diff() { + if ! git diff --exit-code >/dev/null; then + echo "There are still uncommitted changes:" + echo "" + echo "# # # # # # # # # #" + echo "" + git --no-pager diff + echo "" + echo "# # # # # # # # # #" + fi + + echo "All changes compared to develop branch:" + echo "" + echo "# # # # # # # # # #" + echo "" + git --no-pager diff develop + echo "" + echo "# # # # # # # # # #" +} + +finish_release_and_push() { + local CURRENT_VERSION="${1}" + local NEW_RELEASE_VERSION="${2}" + local BASE_RELEASE_VERSION="${3}" + + # Push changes and delete release branch + wait_for_ok "Upgrade from version v${CURRENT_VERSION} to version v${NEW_RELEASE_VERSION} finished. Should the changes be pushed?" + git push origin release/v"${NEW_RELEASE_VERSION}" + + echo "Switching back to develop and deleting branch release/v${NEW_RELEASE_VERSION}..." + + local BASE_DEV_BRANCH_NAME + + if [[ -z "$BASE_RELEASE_VERSION" ]]; then + BASE_DEV_BRANCH_NAME="develop" + else + BASE_DEV_BRANCH_NAME="${BASE_RELEASE_VERSION}/develop" + fi + + git checkout ${BASE_DEV_BRANCH_NAME} + git branch -D release/v"${NEW_RELEASE_VERSION}" +} diff --git a/build/make/self-update.mk b/build/make/self-update.mk index 8eaa807..3bed1ab 100644 --- a/build/make/self-update.mk +++ b/build/make/self-update.mk @@ -1,5 +1,7 @@ +##@ Makefile management + .PHONY: update-makefiles -update-makefiles: do-update-makefiles +update-makefiles: do-update-makefiles ## Update Makefiles to MAKEFILES_VERSION .PHONY: do-update-makefiles do-update-makefiles: $(TMP_DIR) download-and-extract remove-old-files copy-new-files @@ -17,4 +19,14 @@ remove-old-files: .PHONY: copy-new-files copy-new-files: - @cp -r $(TMP_DIR)/makefiles-$(MAKEFILES_VERSION)/build/make $(BUILD_DIR) \ No newline at end of file + @cp -r $(TMP_DIR)/makefiles-$(MAKEFILES_VERSION)/build/make $(BUILD_DIR) + +.PHONY: update-build-libs +update-build-libs: + @echo "Check for newer Build-Lib versions" + build/make/self-update.sh buildlibs + +.PHONY: set-dogu-version +set-dogu-version: + @echo "Set Version of Dogu without Release" + build/make/self-update.sh versions \ No newline at end of file diff --git a/build/make/self-update.sh b/build/make/self-update.sh new file mode 100755 index 0000000..6af39e8 --- /dev/null +++ b/build/make/self-update.sh @@ -0,0 +1,63 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + + +# shellcheck disable=SC1090 +source "$(pwd)/build/make/release_functions.sh" + +TYPE="${1}" + +update_build_libs() { + echo "Get newest version of ces-build-lib and dogu-build-lib" + update_jenkinsfile + echo "Newest Versions set. Please check your Jenkinsfile" +} + +get_highest_version() { + local target="${1}" + local gitCesBuildLib + # getting tags from ces-build.libs OR dogu-build-libs + gitCesBuildLib="$(git ls-remote --tags --refs https://github.com/cloudogu/${target}-build-lib)" + local highest + # Flagfile for getting results out of while-loop + rm -rf .versions + while IFS= read -r line; do + local version + version="$(awk -F'/tags/' '{ for(i=1;i<=NF;i++) print $i }' <<< $line | tail -n 1 | sed 's/[^0-9\.]*//g')" + if [[ $version == *"."* ]] ; then + echo $version >> ".versions" + fi + done <<< "$gitCesBuildLib" + highest=$(sort .versions | tail -n 1) + rm -rf .versions + echo "${highest}" +} + +# Patch Jenkinsfile +update_jenkinsfile() { + sed -i "s/ces-build-lib@[[:digit:]].[[:digit:]].[[:digit:]]/ces-build-lib@$(get_highest_version ces)/g" Jenkinsfile + sed -i "s/dogu-build-lib@v[[:digit:]].[[:digit:]].[[:digit:]]/dogu-build-lib@v$(get_highest_version dogu)/g" Jenkinsfile +} + +# Patch Dogu Version without Release +set_dogu_version() { + CURRENT_TOOL_VERSION=$(get_current_version_by_dogu_json) + echo "$(tput setaf 1)ATTENTION: Make sure that the new version corresponds to the current software version$(tput sgr0)" + NEW_RELEASE_VERSION="$(read_new_version)" + validate_new_version "${NEW_RELEASE_VERSION}" + update_versions "${NEW_RELEASE_VERSION}" +} + +# switch for script entrypoint +if [[ "${TYPE}" == "buildlibs" ]];then + update_build_libs +elif [[ "${TYPE}" == "versions" ]];then + set_dogu_version +else + echo "Unknown target ${TYPE}" +fi + + + diff --git a/build/make/static-analysis.mk b/build/make/static-analysis.mk index b93213c..1c72452 100644 --- a/build/make/static-analysis.mk +++ b/build/make/static-analysis.mk @@ -1,15 +1,19 @@ +##@ Static analysis + STATIC_ANALYSIS_DIR=$(TARGET_DIR)/static-analysis GOIMAGE?=golang -GOTAG?=1.14.13 +GOTAG?=1.25 CUSTOM_GO_MOUNT?=-v /tmp:/tmp REVIEW_DOG=$(TMP_DIR)/bin/reviewdog LINT=$(TMP_DIR)/bin/golangci-lint +LINT_VERSION?=v2.5.0 # ignore tests and mocks -LINTFLAGS=--tests=false --skip-files="^.*_mock.go$$" --skip-files="^.*/mock.*.go$$" +LINTFLAGS=--tests=false --timeout 10m --issues-exit-code 0 +ADDITIONAL_LINTER=-E bodyclose -E containedctx -E contextcheck -E decorder -E dupl -E errname -E forcetypeassert -E funlen -E unparam .PHONY: static-analysis -static-analysis: static-analysis-$(ENVIRONMENT) +static-analysis: static-analysis-$(ENVIRONMENT) ## Start a static analysis of the code .PHONY: static-analysis-ci static-analysis-ci: @@ -39,11 +43,11 @@ $(STATIC_ANALYSIS_DIR)/static-analysis.log: $(STATIC_ANALYSIS_DIR) @echo "" @echo "complete static analysis:" @echo "" - @$(LINT) $(LINTFLAGS) run ./... | tee $@ + @$(LINT) $(LINTFLAGS) run ./... $(ADDITIONAL_LINTER) > $@ $(STATIC_ANALYSIS_DIR)/static-analysis-cs.log: $(STATIC_ANALYSIS_DIR) @echo "run static analysis with export to checkstyle format" - @$(LINT) $(LINTFLAGS) run --out-format=checkstyle ./... > $@ | true + @$(LINT) $(LINTFLAGS) --output.checkstyle.path $@ run ./... $(ADDITIONAL_LINTER) $(STATIC_ANALYSIS_DIR): $(LINT) @mkdir -p $(STATIC_ANALYSIS_DIR) @@ -55,7 +59,8 @@ static-analysis-ci-report-local: $(STATIC_ANALYSIS_DIR)/static-analysis-cs.log $ @cat $(STATIC_ANALYSIS_DIR)/static-analysis-cs.log | $(REVIEW_DOG) -f checkstyle -diff "git diff develop" $(LINT): $(TMP_DIR) - @curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TMP_DIR)/bin v1.33.0 + @echo "Download golangci-lint $(LINT_VERSION)..." + @curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TMP_DIR)/bin $(LINT_VERSION) $(REVIEW_DOG): $(TMP_DIR) - @curl -sfL https://raw.githubusercontent.com/reviewdog/reviewdog/master/install.sh| sh -s -- -b $(TMP_DIR)/bin \ No newline at end of file + @curl -sfL https://raw.githubusercontent.com/reviewdog/reviewdog/master/install.sh| sh -s -- -b $(TMP_DIR)/bin diff --git a/build/make/test-common.mk b/build/make/test-common.mk index d855b35..1946f83 100644 --- a/build/make/test-common.mk +++ b/build/make/test-common.mk @@ -1,2 +1,8 @@ -$(GOPATH)/bin/go-junit-report: - @GO111MODULE=off $(GO_CALL) get -u github.com/jstemmer/go-junit-report +TEST_COMMON_MK_INCLUDE_MARKER="test-common.mk" + +GO_JUNIT_REPORT=$(UTILITY_BIN_PATH)/go-junit-report +GO_JUNIT_REPORT_VERSION=v2.1.0 + +$(GO_JUNIT_REPORT): $(UTILITY_BIN_PATH) + @echo "Download go-junit-report..." + @$(call go-get-tool,$@,github.com/jstemmer/go-junit-report/v2@$(GO_JUNIT_REPORT_VERSION)) diff --git a/build/make/test-integration.mk b/build/make/test-integration.mk index 2bf53c2..cefad99 100644 --- a/build/make/test-integration.mk +++ b/build/make/test-integration.mk @@ -1,12 +1,20 @@ +# this also works with older main Makefiles which include all test*.mk files on top-level. +ifeq (${TEST_COMMON_MK_INCLUDE_MARKER}, ) + include ${BUILD_DIR}/make/test-common.mk +endif + +##@ Integration testing + INTEGRATION_TEST_DIR=$(TARGET_DIR)/integration-tests XUNIT_INTEGRATION_XML=$(INTEGRATION_TEST_DIR)/integration-tests.xml INTEGRATION_TEST_LOG=$(INTEGRATION_TEST_DIR)/integration-tests.log INTEGRATION_TEST_REPORT=$(INTEGRATION_TEST_DIR)/coverage.out PRE_INTEGRATIONTESTS?=start-local-docker-compose POST_INTEGRATIONTESTS?=stop-local-docker-compose +INTEGRATION_TEST_NAME_PATTERN?=.* .PHONY: integration-test -integration-test: $(XUNIT_INTEGRATION_XML) +integration-test: $(XUNIT_INTEGRATION_XML) ## Start integration tests .PHONY: start-local-docker-compose start-local-docker-compose: @@ -27,22 +35,16 @@ else echo "Found CI environment. Nothing to be done" endif -$(XUNIT_INTEGRATION_XML): $(SRC) $(GOPATH)/bin/go-junit-report +$(XUNIT_INTEGRATION_XML): $(SRC) $(GO_JUNIT_REPORT) ifneq ($(strip $(PRE_INTEGRATIONTESTS)),) @make $(PRE_INTEGRATIONTESTS) endif @mkdir -p $(INTEGRATION_TEST_DIR) - @echo 'mode: set' > ${INTEGRATION_TEST_REPORT} + @echo 'mode: set' > $(INTEGRATION_TEST_REPORT) @rm -f $(INTEGRATION_TEST_LOG) || true - @for PKG in $(PACKAGES_FOR_INTEGRATION_TEST) ; do \ - ${GO_CALL} test -tags=${GO_BUILD_TAG_INTEGRATION_TEST} -v $$PKG -coverprofile=${INTEGRATION_TEST_REPORT}.tmp 2>&1 | tee $(INTEGRATION_TEST_LOG).tmp ; \ - cat ${INTEGRATION_TEST_REPORT}.tmp | tail +2 >> ${INTEGRATION_TEST_REPORT} ; \ - rm -f ${INTEGRATION_TEST_REPORT}.tmp ; \ - cat $(INTEGRATION_TEST_LOG).tmp >> $(INTEGRATION_TEST_LOG) ; \ - rm -f $(INTEGRATION_TEST_LOG).tmp ; \ - done - @cat $(INTEGRATION_TEST_LOG) | go-junit-report > $@ + @$(GO_CALL) test ./... -v -tags=${GO_BUILD_TAG_INTEGRATION_TEST} -coverpkg=./... -coverprofile=${INTEGRATION_TEST_REPORT} -run ${INTEGRATION_TEST_NAME_PATTERN} 2>&1 | tee $(INTEGRATION_TEST_LOG) + @cat $(INTEGRATION_TEST_LOG) | $(GO_JUNIT_REPORT) > $@ @if grep '^FAIL' $(INTEGRATION_TEST_LOG); then \ exit 1; \ fi diff --git a/build/make/test-unit.mk b/build/make/test-unit.mk index 8b696ee..03cce8a 100644 --- a/build/make/test-unit.mk +++ b/build/make/test-unit.mk @@ -1,4 +1,11 @@ +ifeq (${TEST_COMMON_MK_INCLUDE_MARKER}, ) + include ${BUILD_DIR}/make/test-common.mk +endif + +##@ Unit testing + UNIT_TEST_DIR=$(TARGET_DIR)/unit-tests +XUNIT_JSON=$(UNIT_TEST_DIR)/report.json XUNIT_XML=$(UNIT_TEST_DIR)/unit-tests.xml UNIT_TEST_LOG=$(UNIT_TEST_DIR)/unit-tests.log COVERAGE_REPORT=$(UNIT_TEST_DIR)/coverage.out @@ -6,10 +13,16 @@ COVERAGE_REPORT=$(UNIT_TEST_DIR)/coverage.out PRE_UNITTESTS?= POST_UNITTESTS?= +ASJSON?= + .PHONY: unit-test -unit-test: $(XUNIT_XML) +unit-test: $(XUNIT_JSON) ## Start unit tests -$(XUNIT_XML): $(SRC) $(GOPATH)/bin/go-junit-report +ifeq ($(ENVIRONMENT),ci) +ASJSON='-json' +endif + +$(XUNIT_JSON): $(SRC) $(GO_JUNIT_REPORT) ifneq ($(strip $(PRE_UNITTESTS)),) @make $(PRE_UNITTESTS) endif @@ -18,13 +31,15 @@ endif @echo 'mode: set' > ${COVERAGE_REPORT} @rm -f $(UNIT_TEST_LOG) || true @for PKG in $(PACKAGES) ; do \ - ${GO_CALL} test -v $$PKG -coverprofile=${COVERAGE_REPORT}.tmp 2>&1 | tee $(UNIT_TEST_LOG).tmp ; \ + ${GO_CALL} test -v $$PKG -coverprofile=${COVERAGE_REPORT}.tmp ${ASJSON} 2>&1 | tee $(UNIT_TEST_LOG).tmp ; \ cat ${COVERAGE_REPORT}.tmp | tail +2 >> ${COVERAGE_REPORT} ; \ rm -f ${COVERAGE_REPORT}.tmp ; \ cat $(UNIT_TEST_LOG).tmp >> $(UNIT_TEST_LOG) ; \ rm -f $(UNIT_TEST_LOG).tmp ; \ done - @cat $(UNIT_TEST_LOG) | go-junit-report > $@ + @cat $(UNIT_TEST_LOG) >> $@ + @cat $(UNIT_TEST_LOG) | $(GO_JUNIT_REPORT) -parser gojson > $(XUNIT_XML) + @if grep '^FAIL' $(UNIT_TEST_LOG); then \ exit 1; \ fi diff --git a/build/make/trivyscan.mk b/build/make/trivyscan.mk new file mode 100644 index 0000000..577853c --- /dev/null +++ b/build/make/trivyscan.mk @@ -0,0 +1,9 @@ +# used to create switch the dogu to a prerelease namespace +# e.g. official/usermgmt -> prerelease_official/usermgmt + +# scan a already build dogu image with trivy +# usage: make trivysan - will scan with severity CRITICAL +# make SEVERITY="HIGH, CRITICAL" trivysacn - will scan with different severity options (e.g. HIGH and CRITICAL) +.PHONY: trivyscan +trivyscan: + build/make/trivyscan.sh scan $(SEVERITY) \ No newline at end of file diff --git a/build/make/trivyscan.sh b/build/make/trivyscan.sh new file mode 100755 index 0000000..1fa050d --- /dev/null +++ b/build/make/trivyscan.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# scan a already build image for CVE findings +# Get tag name from dogu.json +trivy_scan() { + echo "Build image and get Tag-Name:" + IMAGE_TAG="$(jq ".Image" --raw-output dogu.json):$(jq ".Version" --raw-output dogu.json)" + docker run -v /var/run/docker.sock:/var/run/docker.sock aquasec/trivy image --severity $SEVERITY $IMAGE_TAG +} + +TYPE="${1}" +SEVERITY="${2:-"CRITICAL"}" + +if [[ "${TYPE}" == "scan" ]];then + trivy_scan +fi \ No newline at end of file diff --git a/build/make/variables.mk b/build/make/variables.mk index 02ac1ca..5558626 100644 --- a/build/make/variables.mk +++ b/build/make/variables.mk @@ -15,10 +15,11 @@ GO_ENVIRONMENT?= # GO_CALL accomodates the go CLI command as well as necessary environment variables which are optional. GO_CALL=${GO_ENVIRONMENT} go PACKAGES=$(shell ${GO_CALL} list ./... | grep -v /vendor/) -PACKAGES_FOR_INTEGRATION_TEST?=${PACKAGES} GO_BUILD_TAG_INTEGRATION_TEST?=integration +GOMODULES=on +UTILITY_BIN_PATH?=${WORKDIR}/.bin -SRC:=$(shell find "${WORKDIR}" -type f -name "*.go" -not -path "./vendor/*") +SRC:=$(shell find "${WORKDIR}" -type f -name "*.go" -not -path "*/vendor/*") # debian stuff DEBIAN_BUILD_DIR=$(BUILD_DIR)/deb @@ -35,6 +36,7 @@ endif YARN_TARGET=$(WORKDIR)/node_modules BOWER_TARGET?=$(WORKDIR)/public/vendor +NODE_VERSION?=8 UID_NR:=$(shell id -u) GID_NR:=$(shell id -g) @@ -57,3 +59,54 @@ $(PASSWD): $(TMP_DIR) $(ETCGROUP): $(TMP_DIR) @echo "root:x:0:" > $(ETCGROUP) @echo "$(USER):x:$(GID_NR):" >> $(ETCGROUP) + +$(UTILITY_BIN_PATH): + @mkdir -p $@ + +# Subdirectories of workdir where no mocks should be generated. +# Multiple directories can be separated by space, comma or whatever is not a word to regex. +MOCKERY_IGNORED=vendor,build,docs + +##@ General + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +.PHONY: info +info: ## Print build information + @echo "dumping build information ..." + @echo "Version : $(VERSION)" + @echo "Commit-ID : $(COMMIT_ID)" + @echo "Environment: $(ENVIRONMENT)" + @echo "Branch : $(BRANCH)" + @echo "Packages : $(PACKAGES)" + + +# go-get-tool will 'go get' any package $2 and install it to $1. +define go-get-tool + @[ -f $(1) ] || { \ + set -e ;\ + TMP_DIR=$$(mktemp -d) ;\ + cd $$TMP_DIR ;\ + go mod init tmp ;\ + echo "Downloading $(2)" ;\ + GOBIN=$(UTILITY_BIN_PATH) go install $(2) ;\ + rm -rf $$TMP_DIR ;\ + } +endef + +# curl-get-tool-from-tar 'curl get' any source tar $2, sha256 checks with $3 and installs the file path $4 to $1. The intermediate folders from the archive can be stripped with $5 (Use 0 if the binary is in root). +define curl-get-tool-from-tar + @[ -f $(1) ] || { \ + set -e ;\ + echo "Downloading $(2) to $(1)" ;\ + TMP_FILE_PATH="$(TMP_DIR)/$$(basename "$(1)")" ;\ + mkdir -p "$(TMP_DIR)" ;\ + curl -L -s -o "$$TMP_FILE_PATH" "$(2)" ;\ + echo "Checking with sum: $3" ;\ + echo "$(3) $$TMP_FILE_PATH" | sha256sum -c ;\ + echo "Extracting $(4) to $$(dirname $(1))" ;\ + tar -xf $$TMP_FILE_PATH -C $$(dirname $(1)) --strip-components=$(5) $(4) ;\ + } +endef diff --git a/build/make/version-sha.mk b/build/make/version-sha.mk new file mode 100644 index 0000000..1335532 --- /dev/null +++ b/build/make/version-sha.mk @@ -0,0 +1,18 @@ +##@ Version + +# This makefile is used to get the sha256sum of a specific github tag-src.tar.gz or .zip. +# You may set any of the following variables before your make call to change the hash url. + +SHA_SUM_ORGANISATION?="cloudogu" +SHA_SUM_REPOSITORY?="ecosystem" +SHA_SUM_FILE_TYPE?="tar.gz" +SHA_SUM_VERSION?="v20.04.4-2" +SHA_SUM_URL?="https://github.com/${SHA_SUM_ORGANISATION}/${SHA_SUM_REPOSITORY}/archive/refs/tags/${SHA_SUM_VERSION}.${SHA_SUM_FILE_TYPE}" + +.PHONY: sha-sum +sha-sum: ## Print out the version + @echo "Downloading from: ${SHA_SUM_URL}" + @wget -O - -o /dev/null "${SHA_SUM_URL}" > .download.for.hash \ + || (echo "Could not be downloaded" && exit 1) \ + && cat .download.for.hash | sha256sum + @rm -f .download.for.hash diff --git a/build/make/vulnerability-scan.mk b/build/make/vulnerability-scan.mk new file mode 100644 index 0000000..5698206 --- /dev/null +++ b/build/make/vulnerability-scan.mk @@ -0,0 +1,13 @@ +##@ Vulnerability scan + +GOVULNCHECK_BIN=${UTILITY_BIN_PATH}/govulncheck +GOVULNCHECK_VERSION?=latest + +${GOVULNCHECK_BIN}: ${UTILITY_BIN_PATH} + $(call go-get-tool,$(GOVULNCHECK_BIN),golang.org/x/vuln/cmd/govulncheck@$(GOVULNCHECK_VERSION)) + +.PHONY: govulncheck +govulncheck: ${GOVULNCHECK_BIN} ## This target is used to scan the go repository against known vulnerabilities + @echo "Start vulnerability against repository" + ${GOVULNCHECK_BIN} -show verbose ./... + @echo "Finished scan" \ No newline at end of file diff --git a/build/make/yarn.mk b/build/make/yarn.mk index 2af3112..6ff7de9 100644 --- a/build/make/yarn.mk +++ b/build/make/yarn.mk @@ -1,15 +1,9 @@ +##@ Yarn dependency management + YARN_LOCK=$(WORKDIR)/yarn.lock .PHONY: yarn-install -yarn-install: $(YARN_TARGET) - -ifeq ($(ENVIRONMENT), ci) - -$(YARN_TARGET): $(YARN_LOCK) - @echo "Yarn install on CI server" - @yarn install - -else +yarn-install: $(YARN_TARGET) ## Execute yarn install $(YARN_TARGET): $(YARN_LOCK) $(PASSWD) @echo "Executing yarn..." @@ -18,8 +12,28 @@ $(YARN_TARGET): $(YARN_LOCK) $(PASSWD) -v $(PASSWD):/etc/passwd:ro \ -v $(WORKDIR):$(WORKDIR) \ -w $(WORKDIR) \ - node:8 \ + node:$(NODE_VERSION) \ yarn install @touch $@ -endif +.PHONY yarn-publish-ci: +yarn-publish-ci: ## Execute yarn publish with '--non-interactive' flag to suppress the version prompt + @echo "Executing yarn publish..." + @docker run --rm \ + -u "$(UID_NR):$(GID_NR)" \ + -v $(PASSWD):/etc/passwd:ro \ + -v $(WORKDIR):$(WORKDIR) \ + -w $(WORKDIR) \ + node:$(NODE_VERSION) \ + yarn publish --non-interactive + +.PHONY yarn-publish: ## Execute yarn publish +yarn-publish: $(YARN_BUILD_TARGET) + @echo "Executing yarn publish..." + @docker run --rm \ + -u "$(UID_NR):$(GID_NR)" \ + -v $(PASSWD):/etc/passwd:ro \ + -v $(WORKDIR):$(WORKDIR) \ + -w $(WORKDIR) \ + node:$(NODE_VERSION) \ + yarn publish diff --git a/docs/container_building_de.md b/docs/container_building_de.md new file mode 100644 index 0000000..6877303 --- /dev/null +++ b/docs/container_building_de.md @@ -0,0 +1,25 @@ +# Containerbau + +Dieses Container-Image bildet die Grundlage für viele Dogu-Container-Images. + +## Anleitung zum Bauen und Bereitstellen + +Auf einem Entwicklungs-Branch: + +1. Aktualisiere die `Makefile` Felder `JAVA_VERSION`, `JAVA_ALPINE_VERSION` und `CHANGE_COUNTER` entsprechend. +2. PR/Merge den Entwicklungs-Branch in den Ziel-Branch. +3. Tagge den Ziel-Commit (z.B. `v3.45.6-7`) für den Release. + +In der Jenkins Pipeline sind folgende Parameter verfügbar: +- `PublishRelease` +- `PublishPrerelease` + +Werden diese Parameter aktiviert, wird das gebaute Image anschließend veröffentlicht. + +Mit aktiviertem `PublishPrerelease` Parameter wird das Image im Namespace `registry.cloudogu.com/prerelease_official/` veröffentlicht. + +Mit aktiviertem `PublishRelease` Parameter wird das Image im Namespace `registry.cloudogu.com/official/` veröffentlicht und es wird ein GitHub Release erstellt. + +Um ältere Varianten des Images erneut zu bauen und zu veröffentlichen, stehen +Branches zu Verfügung, für welche der Build- & Release-Prozess mittels +Parameter analog zum Haupt-Branch gestartet werden kann. diff --git a/docs/container_building_en.md b/docs/container_building_en.md new file mode 100644 index 0000000..c5850d3 --- /dev/null +++ b/docs/container_building_en.md @@ -0,0 +1,23 @@ +# Container building + +This container image provides the base of many dogu container images. + +## Instructions for building and deploying + +On a development branch: + +1. Update the `Makefile` fields `JAVA_VERSION`, `JAVA_ALPINE_VERSION` and `CHANGE_COUNTER` accordingly +2. PR/merge the development branch into the target-branch +3. Tag the target commit (e.g. `v3.45.6-7`) for the release. + +The following parameters are available in the Jenkins Pipeline: +- `PublishRelease` +- `PublishPrerelease` + +If these parameters are enabled, the image will be published after successful build. + +With the `PublishPrerelease` parameter enabled, the image will be published in the namespace `registry.cloudogu.com/prerelease_official/`. + +With the `PublishRelease` parameter enabled, the image will be published in the namespace `registry.cloudogu.com/official/` and a GitHub Release will be created. + +To rebuild and publish older versions of the image, branches are available for which the build and release process can be started using parameters similar to the main branch. diff --git a/unitTests/Dockerfile b/unitTests/Dockerfile index c115b62..057d748 100644 --- a/unitTests/Dockerfile +++ b/unitTests/Dockerfile @@ -1,8 +1,10 @@ ARG BATS_BASE_IMAGE ARG BATS_TAG -FROM ${BATS_BASE_IMAGE}:${BATS_TAG} +FROM ${BATS_BASE_IMAGE:-bats/bats}:${BATS_TAG:-1.11.0} # Make bash more findable by scripts and tests RUN apk add make git bash +# suppress git "detected dubious ownership" error/warning for repos which are checked out later +RUN git config --global --add safe.directory /workspace COPY ./usr/bin/create-ca-certificates.sh /usr/bin/create-ca-certificates.sh \ No newline at end of file