xref: /freebsd/contrib/jemalloc/scripts/gen_travis.py (revision 8ebb3de0c9dfb1a15bf24dcb0ca65cc91e7ad0e8) 
1#!/usr/bin/env python3
2
3from itertools import combinations, chain
4from enum import Enum, auto
5
6
7LINUX = 'linux'
8OSX = 'osx'
9WINDOWS = 'windows'
10FREEBSD = 'freebsd'
11
12
13AMD64 = 'amd64'
14ARM64 = 'arm64'
15PPC64LE = 'ppc64le'
16
17
18TRAVIS_TEMPLATE = """\
19# This config file is generated by ./scripts/gen_travis.py.
20# Do not edit by hand.
21
22# We use 'minimal', because 'generic' makes Windows VMs hang at startup. Also
23# the software provided by 'generic' is simply not needed for our tests.
24# Differences are explained here:
25# https://docs.travis-ci.com/user/languages/minimal-and-generic/
26language: minimal
27dist: focal
28
29jobs:
30  include:
31{jobs}
32
33before_install:
34  - |-
35    if test -f "./scripts/$TRAVIS_OS_NAME/before_install.sh"; then
36      source ./scripts/$TRAVIS_OS_NAME/before_install.sh
37    fi
38
39before_script:
40  - |-
41    if test -f "./scripts/$TRAVIS_OS_NAME/before_script.sh"; then
42      source ./scripts/$TRAVIS_OS_NAME/before_script.sh
43    else
44      scripts/gen_travis.py > travis_script && diff .travis.yml travis_script
45      autoconf
46      # If COMPILER_FLAGS are not empty, add them to CC and CXX
47      ./configure ${{COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" \
48CXX="$CXX $COMPILER_FLAGS"}} $CONFIGURE_FLAGS
49      make -j3
50      make -j3 tests
51    fi
52
53script:
54  - |-
55    if test -f "./scripts/$TRAVIS_OS_NAME/script.sh"; then
56      source ./scripts/$TRAVIS_OS_NAME/script.sh
57    else
58      make check
59    fi
60"""
61
62
63class Option(object):
64    class Type:
65        COMPILER = auto()
66        COMPILER_FLAG = auto()
67        CONFIGURE_FLAG = auto()
68        MALLOC_CONF = auto()
69        FEATURE = auto()
70
71    def __init__(self, type, value):
72        self.type = type
73        self.value = value
74
75    @staticmethod
76    def as_compiler(value):
77        return Option(Option.Type.COMPILER, value)
78
79    @staticmethod
80    def as_compiler_flag(value):
81        return Option(Option.Type.COMPILER_FLAG, value)
82
83    @staticmethod
84    def as_configure_flag(value):
85        return Option(Option.Type.CONFIGURE_FLAG, value)
86
87    @staticmethod
88    def as_malloc_conf(value):
89        return Option(Option.Type.MALLOC_CONF, value)
90
91    @staticmethod
92    def as_feature(value):
93        return Option(Option.Type.FEATURE, value)
94
95    def __eq__(self, obj):
96        return (isinstance(obj, Option) and obj.type == self.type
97                and obj.value == self.value)
98
99
100# The 'default' configuration is gcc, on linux, with no compiler or configure
101# flags.  We also test with clang, -m32, --enable-debug, --enable-prof,
102# --disable-stats, and --with-malloc-conf=tcache:false.  To avoid abusing
103# travis though, we don't test all 2**7 = 128 possible combinations of these;
104# instead, we only test combinations of up to 2 'unusual' settings, under the
105# hope that bugs involving interactions of such settings are rare.
106MAX_UNUSUAL_OPTIONS = 2
107
108
109GCC = Option.as_compiler('CC=gcc CXX=g++')
110CLANG = Option.as_compiler('CC=clang CXX=clang++')
111CL = Option.as_compiler('CC=cl.exe CXX=cl.exe')
112
113
114compilers_unusual = [CLANG,]
115
116
117CROSS_COMPILE_32BIT = Option.as_feature('CROSS_COMPILE_32BIT')
118feature_unusuals = [CROSS_COMPILE_32BIT]
119
120
121configure_flag_unusuals = [Option.as_configure_flag(opt) for opt in (
122    '--enable-debug',
123    '--enable-prof',
124    '--disable-stats',
125    '--disable-libdl',
126    '--enable-opt-safety-checks',
127    '--with-lg-page=16',
128)]
129
130
131malloc_conf_unusuals = [Option.as_malloc_conf(opt) for opt in (
132    'tcache:false',
133    'dss:primary',
134    'percpu_arena:percpu',
135    'background_thread:true',
136)]
137
138
139all_unusuals = (compilers_unusual + feature_unusuals
140    + configure_flag_unusuals + malloc_conf_unusuals)
141
142
143def get_extra_cflags(os, compiler):
144    if os == FREEBSD:
145        return []
146
147    if os == WINDOWS:
148        # For non-CL compilers under Windows (for now it's only MinGW-GCC),
149        # -fcommon needs to be specified to correctly handle multiple
150        # 'malloc_conf' symbols and such, which are declared weak under Linux.
151        # Weak symbols don't work with MinGW-GCC.
152        if compiler != CL.value:
153            return ['-fcommon']
154        else:
155            return []
156
157    # We get some spurious errors when -Warray-bounds is enabled.
158    extra_cflags = ['-Werror', '-Wno-array-bounds']
159    if compiler == CLANG.value or os == OSX:
160        extra_cflags += [
161            '-Wno-unknown-warning-option',
162            '-Wno-ignored-attributes'
163        ]
164    if os == OSX:
165        extra_cflags += [
166            '-Wno-deprecated-declarations',
167        ]
168    return extra_cflags
169
170
171# Formats a job from a combination of flags
172def format_job(os, arch, combination):
173    compilers = [x.value for x in combination if x.type == Option.Type.COMPILER]
174    assert(len(compilers) <= 1)
175    compiler_flags = [x.value for x in combination if x.type == Option.Type.COMPILER_FLAG]
176    configure_flags = [x.value for x in combination if x.type == Option.Type.CONFIGURE_FLAG]
177    malloc_conf = [x.value for x in combination if x.type == Option.Type.MALLOC_CONF]
178    features = [x.value for x in combination if x.type == Option.Type.FEATURE]
179
180    if len(malloc_conf) > 0:
181        configure_flags.append('--with-malloc-conf=' + ','.join(malloc_conf))
182
183    if not compilers:
184        compiler = GCC.value
185    else:
186        compiler = compilers[0]
187
188    extra_environment_vars = ''
189    cross_compile = CROSS_COMPILE_32BIT.value in features
190    if os == LINUX and cross_compile:
191        compiler_flags.append('-m32')
192
193    features_str = ' '.join([' {}=yes'.format(feature) for feature in features])
194
195    stringify = lambda arr, name: ' {}="{}"'.format(name, ' '.join(arr)) if arr else ''
196    env_string = '{}{}{}{}{}{}'.format(
197            compiler,
198            features_str,
199            stringify(compiler_flags, 'COMPILER_FLAGS'),
200            stringify(configure_flags, 'CONFIGURE_FLAGS'),
201            stringify(get_extra_cflags(os, compiler), 'EXTRA_CFLAGS'),
202            extra_environment_vars)
203
204    job = '    - os: {}\n'.format(os)
205    job += '      arch: {}\n'.format(arch)
206    job += '      env: {}'.format(env_string)
207    return job
208
209
210def generate_unusual_combinations(unusuals, max_unusual_opts):
211    """
212    Generates different combinations of non-standard compilers, compiler flags,
213    configure flags and malloc_conf settings.
214
215    @param max_unusual_opts: Limit of unusual options per combination.
216    """
217    return chain.from_iterable(
218            [combinations(unusuals, i) for i in range(max_unusual_opts + 1)])
219
220
221def included(combination, exclude):
222    """
223    Checks if the combination of options should be included in the Travis
224    testing matrix.
225
226    @param exclude: A list of options to be avoided.
227    """
228    return not any(excluded in combination for excluded in exclude)
229
230
231def generate_jobs(os, arch, exclude, max_unusual_opts, unusuals=all_unusuals):
232    jobs = []
233    for combination in generate_unusual_combinations(unusuals, max_unusual_opts):
234        if included(combination, exclude):
235            jobs.append(format_job(os, arch, combination))
236    return '\n'.join(jobs)
237
238
239def generate_linux(arch):
240    os = LINUX
241
242    # Only generate 2 unusual options for AMD64 to reduce matrix size
243    max_unusual_opts = MAX_UNUSUAL_OPTIONS if arch == AMD64 else 1
244
245    exclude = []
246    if arch == PPC64LE:
247        # Avoid 32 bit builds and clang on PowerPC
248        exclude = (CROSS_COMPILE_32BIT, CLANG,)
249
250    return generate_jobs(os, arch, exclude, max_unusual_opts)
251
252
253def generate_macos(arch):
254    os = OSX
255
256    max_unusual_opts = 1
257
258    exclude = ([Option.as_malloc_conf(opt) for opt in (
259            'dss:primary',
260            'percpu_arena:percpu',
261            'background_thread:true')] +
262        [Option.as_configure_flag('--enable-prof')] +
263        [CLANG,])
264
265    return generate_jobs(os, arch, exclude, max_unusual_opts)
266
267
268def generate_windows(arch):
269    os = WINDOWS
270
271    max_unusual_opts = 3
272    unusuals = (
273        Option.as_configure_flag('--enable-debug'),
274        CL,
275        CROSS_COMPILE_32BIT,
276    )
277    return generate_jobs(os, arch, (), max_unusual_opts, unusuals)
278
279
280def generate_freebsd(arch):
281    os = FREEBSD
282
283    max_unusual_opts = 4
284    unusuals = (
285        Option.as_configure_flag('--enable-debug'),
286        Option.as_configure_flag('--enable-prof --enable-prof-libunwind'),
287        Option.as_configure_flag('--with-lg-page=16 --with-malloc-conf=tcache:false'),
288        CROSS_COMPILE_32BIT,
289    )
290    return generate_jobs(os, arch, (), max_unusual_opts, unusuals)
291
292
293
294def get_manual_jobs():
295    return """\
296    # Development build
297    - os: linux
298      env: CC=gcc CXX=g++ CONFIGURE_FLAGS="--enable-debug \
299--disable-cache-oblivious --enable-stats --enable-log --enable-prof" \
300EXTRA_CFLAGS="-Werror -Wno-array-bounds"
301    # --enable-expermental-smallocx:
302    - os: linux
303      env: CC=gcc CXX=g++ CONFIGURE_FLAGS="--enable-debug \
304--enable-experimental-smallocx --enable-stats --enable-prof" \
305EXTRA_CFLAGS="-Werror -Wno-array-bounds"
306"""
307
308
309def main():
310    jobs = '\n'.join((
311        generate_windows(AMD64),
312
313        generate_freebsd(AMD64),
314
315        generate_linux(AMD64),
316        generate_linux(PPC64LE),
317
318        generate_macos(AMD64),
319
320        get_manual_jobs(),
321    ))
322
323    print(TRAVIS_TEMPLATE.format(jobs=jobs))
324
325
326if __name__ == '__main__':
327    main()
328