Paul Bastide

Automation and Build Tips

Solution: Host Override using Curl

curl -H "Host: <hostname>" http://<ip>:9080/path-to-api

The hostname should be the remote domain name and the IP should be the node specific IP.

Links

Travis APIs – LINTING the Travis Yaml

To lint the travis.yml run the following

Shell

curl -X POST https://api.travis-ci.com/lint -H "Accept: application/vnd.travis-ci.2+json" -H "Authorization: token <MYTOKEN>" --data-binary "@./.travis.yml" -v

Example Response

{"lint":{"warnings":[{"key":["deploy"],"message":"dropping \"file\" section: unexpected sequence"},{"key":[],"message":"value for \"addons\" section is empty, dropping"},{"key":["addons"],"message":"unexpected key \"apt\", dropping"}]}}

More detail at https://developer.travis-ci.com/resource/lint#Lint

Finding a File in an RPM via Java

I had to find a file in an RPM using Java. It was a really handy learning experience so I used the open source libraries which support a developer compressing and opening cpio, rpm files. Commons IO has a nice IOUtils feature which I used for ease of use.

Download the files from Apache Commons IO Apache Commons Compress Redline RPM

There are a lot of tips included in the following gist:

package demo;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.channels.Channels;
import java.util.zip.GZIPInputStream;
import org.apache.commons.compress.archivers.cpio.CpioArchiveEntry;
import org.apache.commons.compress.archivers.cpio.CpioArchiveInputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorInputStream;
import org.apache.commons.io.IOUtils;
import org.redline_rpm.ReadableChannelWrapper;
import org.redline_rpm.Scanner;
import org.redline_rpm.header.AbstractHeader.Entry;
import org.redline_rpm.header.Format;
import org.redline_rpm.header.Header;
import org.redline_rpm.header.Header.HeaderTag;
import org.redline_rpm.header.PayloadCompressionType;
public class Runner {
private final static String defaultUrl = "my.rpm;"
/**
* @param args
*/
public static void main(String[] args) {
String url = defaultUrl;
if (args.length > 1) {
url = defaultUrl;
}
/**
* Checks if the File Exists, and manages the download if it does not
* alread exists
*/
System.out.println("= Download the RPM =");
File rpm = new File(generateRPMNameFromURL(url));
if (!rpm.exists()) {
boolean success = downloadFile(url);
if (!success) {
System.out
.println("WARNING: No Download Completed - Refer to StackTrace");
}
}
/**
*
*/
System.out.println("= Begin Dump =");
File cpio = new File(rpm.getPath() + ".cpio");
if (!cpio.exists()) // Else dump it
dumpToCpio(rpm);
listCpioFiles(cpio);
}
/**
* lists the files in the CPIO
*
* @param rpm
*/
public static void listCpioFiles(File cpio) {
try {
CpioArchiveInputStream cpioIn = new CpioArchiveInputStream(
new FileInputStream(cpio));
CpioArchiveEntry cpioEntry;
while ((cpioEntry = (CpioArchiveEntry) cpioIn.getNextEntry()) != null) {
outputEntry(cpioEntry.getName(), cpioEntry, cpioIn);
}
cpioIn.close();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* outputs the file name if it's worth while to output
*
* @param fileName
* @throws IOException
*/
public static void outputEntry(String fileName, CpioArchiveEntry entry,
CpioArchiveInputStream cpioIn) throws IOException {
if (fileName.startsWith("target.xml") {
String outputName = fileName.substring(fileName.lastIndexOf("/"));
System.out.println("/" + outputName);
File cpioFile = new File("/" + outputName);
OutputStream fos = new FileOutputStream(cpioFile);
IOUtils.copy(cpioIn, fos);
System.out.println("DONE");
fos.close();
}
}
/**
* dumps the rpm to a cpio file
*
* @param rpm
*/
public static void dumpToCpio(File rpm) {
try {
InputStream fios = new FileInputStream(rpm.getPath());
ReadableChannelWrapper in = new ReadableChannelWrapper(
Channels.newChannel(fios));
Scanner scanner = new Scanner();
Format format = scanner.run(in);
Header rpmheader = format.getHeader();
InputStream uncompressed = Runner
.openPayloadStream(rpmheader, fios);
File cpio = new File(rpm.getPath() + ".cpio");
OutputStream fos = new FileOutputStream(cpio);
IOUtils.copy(uncompressed, fos);
fos.close();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* Create the proper stream wrapper to handling the rpmIS payload section
* based on the payload compression header tag.
*
* @param header
* the header
* @param rpmIS
* raw input stream of the rpm
* @return the "proper" input stream
* @throws IOException
* an IO error occurred
*/
public static InputStream openPayloadStream(Header header, InputStream rpmIS)
throws IOException {
Entry pcEntry = header.getEntry(HeaderTag.PAYLOADCOMPRESSOR);
String[] pc = (String[]) pcEntry.getValues();
PayloadCompressionType pcType = PayloadCompressionType.valueOf(pc[0]);
InputStream payloadIS = rpmIS;
switch (pcType) {
case none:
break;
case gzip:
payloadIS = new GZIPInputStream(rpmIS);
break;
case bzip2:
try {
payloadIS = new org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream(
rpmIS);
} catch (Exception e) {
throw new IOException(
"Failed to load BZIP2 compression stream", e);
}
break;
case xz:
try {
payloadIS = new XZCompressorInputStream(rpmIS);
} catch (Exception e) {
throw new IOException("Failed to load XZ compression stream", e);
}
break;
}
return payloadIS;
}
/**
* download the file at the given url only supports http and valid https
* certificates
*
* @param url
* @return
*/
public static boolean downloadFile(String u) {
boolean downloaded = false;
try {
URL url = new URL(u);
HttpURLConnection httpCon = (HttpURLConnection) url
.openConnection();
httpCon.setRequestMethod("GET");
InputStream is = httpCon.getInputStream();
System.out.println("Response Code " + httpCon.getResponseCode());
File rpm = new File(generateRPMNameFromURL(u));
OutputStream fos = new FileOutputStream(rpm);
IOUtils.copy(is, fos);
fos.close();
downloaded = true;
} catch (Exception e) {
e.printStackTrace();
}
return downloaded;
}
/**
* generates RPM Name from URL
*
* @param url
* @return
*/
private static String generateRPMNameFromURL(String url) {
return url.substring(url.lastIndexOf("/"));
}
view raw Runner.java hosted with ❤ by GitHub

GitHub Action: Skip on Label

In your GitHub Action, use if: "!contains(github.event.head_commit.message, 'ci skip')"

You can see an example at this job

An example copied below

jobs:
e2e-persistence:
runs-on: ubuntu-latest
if: "!contains(github.event.pull_request.labels.*.name, 'ci-skip')"
strategy:
matrix:
java: [ 'openjdk11' ]
persistence: [ 'postgres' ]
fail-fast: false
steps:

Primer: Tips for using Random Test Files

On Mac and Linux, create a random file

dd if=/dev/urandom of=random-test-file-100m bs=1024k count=100

Example:

$ dd if=/dev/urandom of=random-test-file-100m bs=1024k count=100
100+0 records in
100+0 records out
102400000 bytes transferred in 10.143507 secs (10095128 bytes/sec)

This is a random file. You should also use shasum -A 256 and make sure the file you are using for test is what you get back. Do also remember 1024 bytes to a KB… use blocksizes of 1024

On linux, create a fast file:

fallocate -l 100M test-file-100m

This takes less than a second. The file is not good for testing random behaviors with large files as the generated file is huge, but the same bytes repated for 100M.

To check the size of the sample files, you can use human readable format settings.

MacBook-Air:test user$ ls -lSh
total 24
-rw-r–r– 1 user staff 10K Dec 20 19:53 setup.log

VirtualBox Installation Issue – Code Signer

While installing virtualbox, I ran into this issue Kernel Install Error

Net: the Oracle/VirtualBox ID is NOT in the approved keys/certs.

I did not follow the exact steps.

Launch Terminal sudo -s

Enter password for the user to switch spctl kext-consent –disable

Install VirtualBox spctl kext-consent –enable

At the end of the steps, I was able to launch VirtualBox. I did not have to reboot.

Solution: Control M character and Coalesce

How to remove CTRL-M (^M) characters from a file in Linux

Removing the ^M character in VIM / VI

:%s/^M//g

If you do end up taking bad data, Coalesce when you pull it out to clear it out. When you have an unexpected bad data coalesce it. IF NULL, THEN the alternative in COALESCE

Solution: Automation: Virtual Box and Cisco

I ran into this issue with CISCO VPN and VirtualBox where the VBox is unable to resolve DNS on the VPN of the host

$ VBoxManage list vms
“Data Stage Server” {e94cc386-9ab9-476c-8dfa-fa877ffa0c8c}
$ VBoxManage modifyvm “Data Stage Server” –natdnshostresolver1 on

Solution: Locked out of Terminal Services

<bang-head/> I locked my team out of a terminal services session when I stopped the terminal server. I am on a Mac and could not restart the service using the MMC snap-in.

I installed Samba4 – samba4-common and used the net rpc service start on TermService and bam… back in business.

[root@hostname ~]# net rpc service start TermService -I 1.1.1.1 --user Administrator
Enter Administrator's password:
...
Successfully started service: TermService

Links

Jenkinsfile Triggers

It took me far too long to get Jenkinsfile to stop overwriting my Triggers as noted in issue 692. You’ll see:

WARNING: The properties step will remove all JobPropertys currently 
configured in this job, either from the UI or from an earlier properties 
step.
#!groovy
pipeline {
// Run on any jenkins agent
agent any
// Fix the versions of Java and Maven.
// Adds to the path to directly call maven and java
tools {
maven 'maven-3.3.9'
jdk 'java-x86_64-80'
}
//https://github.com/jenkinsci/gitlab-plugin
triggers {
gitlab(
triggerOnPush: false,
triggerOnMergeRequest: true, triggerOpenMergeRequestOnPush: "never",
triggerOnNoteRequest: true,
noteRegex: "Jenkins please retry a build",
skipWorkInProgressMergeRequest: true,
ciSkip: true,
setBuildDescription: true,
addNoteOnMergeRequest: true,
addCiMessage: true,
addVoteOnMergeRequest: true,
acceptMergeRequestOnSuccess: false,
branchFilterType: "NameBasedFilter",
includeBranchesSpec: 'development',
excludeBranchesSpec: "",
secretToken: 'abcdefghijklmnopqrst1234'
)
}
stages {
stage('API') {
steps {
script{
// Build
wrap([$class: 'AnsiColorBuildWrapper']) {
sh "mvn clean package -f api/pom.xml"
jacoco()
}
}
}
post {
success {
updateGitlabCommitStatus name: 'jenkins', state: 'success'
}
failure {
updateGitlabCommitStatus name: 'jenkins', state: 'failed'
}
}
}
}
options {
buildDiscarder(logRotator(numToKeepStr:'10'))
disableConcurrentBuilds()
timeout(time: 60, unit: 'MINUTES')
}
}
view raw Jenkinsfile hosted with ❤ by GitHub

References
https://github.com/jenkinsci/gitlab-plugin
https://github.com/jenkinsci/gitlab-plugin/blob/master/src/main/java/com/dabsquared/gitlabjenkins/GitLabPushTrigger.java
https://github.com/jenkinsci/gitlab-plugin/blob/master/src/main/resources/com/dabsquared/gitlabjenkins/GitLabPushTrigger/config.jelly
https://github.com/jenkinsci/job-dsl-plugin/blob/master/job-dsl-core/src/main/groovy/javaposse/jobdsl/dsl/helpers/triggers/GitLabTriggerContext.groovy
https://github.com/jenkinsci/pipeline-model-definition-plugin/wiki/Trigger-runs
https://dev.to/pencillr/jenkins-pipelines-and-their-dirty-secrets-2
https://github.com/jenkinsci/pipeline-model-definition-plugin/wiki/Parametrized-pipelines
https://issues.jenkins-ci.org/browse/JENKINS-45053
https://jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline
https://jenkins.io/doc/pipeline/steps/
https://gitlab.switch.ch/etienne.dysli-metref/idpv3-mfa/commit/6d2974b1199fb2101f5f4299a974cac66a220080
https://github.com/jenkinsci/gitlab-plugin/issues/417