xref: /freebsd/contrib/jemalloc/scripts/gen_travis.py (revision b5daf675efc746611c7cfcd1fa474b8905064c4b) 
1<<<<<<< HEAD
2#!/usr/bin/env python3
3
4from itertools import combinations, chain
5from enum import Enum, auto
6
7
8LINUX = 'linux'
9OSX = 'osx'
10WINDOWS = 'windows'
11FREEBSD = 'freebsd'
12
13
14AMD64 = 'amd64'
15ARM64 = 'arm64'
16PPC64LE = 'ppc64le'
17
18
19TRAVIS_TEMPLATE = """\
20# This config file is generated by ./scripts/gen_travis.py.
21# Do not edit by hand.
22
23# We use 'minimal', because 'generic' makes Windows VMs hang at startup. Also
24# the software provided by 'generic' is simply not needed for our tests.
25# Differences are explained here:
26# https://docs.travis-ci.com/user/languages/minimal-and-generic/
27language: minimal
28dist: focal
29
30jobs:
31  include:
32{jobs}
33
34before_install:
35  - |-
36    if test -f "./scripts/$TRAVIS_OS_NAME/before_install.sh"; then
37      source ./scripts/$TRAVIS_OS_NAME/before_install.sh
38    fi
39
40before_script:
41  - |-
42    if test -f "./scripts/$TRAVIS_OS_NAME/before_script.sh"; then
43      source ./scripts/$TRAVIS_OS_NAME/before_script.sh
44    else
45      scripts/gen_travis.py > travis_script && diff .travis.yml travis_script
46      autoconf
47      # If COMPILER_FLAGS are not empty, add them to CC and CXX
48      ./configure ${{COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" \
49CXX="$CXX $COMPILER_FLAGS"}} $CONFIGURE_FLAGS
50      make -j3
51      make -j3 tests
52    fi
53
54script:
55  - |-
56    if test -f "./scripts/$TRAVIS_OS_NAME/script.sh"; then
57      source ./scripts/$TRAVIS_OS_NAME/script.sh
58    else
59      make check
60    fi
61"""
62
63
64class Option(object):
65    class Type:
66        COMPILER = auto()
67        COMPILER_FLAG = auto()
68        CONFIGURE_FLAG = auto()
69        MALLOC_CONF = auto()
70        FEATURE = auto()
71
72    def __init__(self, type, value):
73        self.type = type
74        self.value = value
75
76    @staticmethod
77    def as_compiler(value):
78        return Option(Option.Type.COMPILER, value)
79
80    @staticmethod
81    def as_compiler_flag(value):
82        return Option(Option.Type.COMPILER_FLAG, value)
83
84    @staticmethod
85    def as_configure_flag(value):
86        return Option(Option.Type.CONFIGURE_FLAG, value)
87
88    @staticmethod
89    def as_malloc_conf(value):
90        return Option(Option.Type.MALLOC_CONF, value)
91
92    @staticmethod
93    def as_feature(value):
94        return Option(Option.Type.FEATURE, value)
95
96    def __eq__(self, obj):
97        return (isinstance(obj, Option) and obj.type == self.type
98                and obj.value == self.value)
99
100
101# The 'default' configuration is gcc, on linux, with no compiler or configure
102# flags.  We also test with clang, -m32, --enable-debug, --enable-prof,
103# --disable-stats, and --with-malloc-conf=tcache:false.  To avoid abusing
104# travis though, we don't test all 2**7 = 128 possible combinations of these;
105# instead, we only test combinations of up to 2 'unusual' settings, under the
106# hope that bugs involving interactions of such settings are rare.
107MAX_UNUSUAL_OPTIONS = 2
108
109
110GCC = Option.as_compiler('CC=gcc CXX=g++')
111CLANG = Option.as_compiler('CC=clang CXX=clang++')
112CL = Option.as_compiler('CC=cl.exe CXX=cl.exe')
113
114
115compilers_unusual = [CLANG,]
116
117
118CROSS_COMPILE_32BIT = Option.as_feature('CROSS_COMPILE_32BIT')
119feature_unusuals = [CROSS_COMPILE_32BIT]
120
121
122configure_flag_unusuals = [Option.as_configure_flag(opt) for opt in (
123    '--enable-debug',
124    '--enable-prof',
125    '--disable-stats',
126    '--disable-libdl',
127    '--enable-opt-safety-checks',
128    '--with-lg-page=16',
129)]
130
131
132malloc_conf_unusuals = [Option.as_malloc_conf(opt) for opt in (
133    'tcache:false',
134    'dss:primary',
135    'percpu_arena:percpu',
136    'background_thread:true',
137)]
138
139
140all_unusuals = (compilers_unusual + feature_unusuals
141    + configure_flag_unusuals + malloc_conf_unusuals)
142
143
144def get_extra_cflags(os, compiler):
145    if os == FREEBSD:
146        return []
147
148    if os == WINDOWS:
149        # For non-CL compilers under Windows (for now it's only MinGW-GCC),
150        # -fcommon needs to be specified to correctly handle multiple
151        # 'malloc_conf' symbols and such, which are declared weak under Linux.
152        # Weak symbols don't work with MinGW-GCC.
153        if compiler != CL.value:
154            return ['-fcommon']
155        else:
156            return []
157
158    # We get some spurious errors when -Warray-bounds is enabled.
159    extra_cflags = ['-Werror', '-Wno-array-bounds']
160    if compiler == CLANG.value or os == OSX:
161        extra_cflags += [
162            '-Wno-unknown-warning-option',
163            '-Wno-ignored-attributes'
164        ]
165    if os == OSX:
166        extra_cflags += [
167            '-Wno-deprecated-declarations',
168        ]
169    return extra_cflags
170
171
172# Formats a job from a combination of flags
173def format_job(os, arch, combination):
174    compilers = [x.value for x in combination if x.type == Option.Type.COMPILER]
175    assert(len(compilers) <= 1)
176    compiler_flags = [x.value for x in combination if x.type == Option.Type.COMPILER_FLAG]
177    configure_flags = [x.value for x in combination if x.type == Option.Type.CONFIGURE_FLAG]
178    malloc_conf = [x.value for x in combination if x.type == Option.Type.MALLOC_CONF]
179    features = [x.value for x in combination if x.type == Option.Type.FEATURE]
180
181    if len(malloc_conf) > 0:
182        configure_flags.append('--with-malloc-conf=' + ','.join(malloc_conf))
183
184    if not compilers:
185        compiler = GCC.value
186    else:
187        compiler = compilers[0]
188
189    extra_environment_vars = ''
190    cross_compile = CROSS_COMPILE_32BIT.value in features
191    if os == LINUX and cross_compile:
192        compiler_flags.append('-m32')
193
194    features_str = ' '.join([' {}=yes'.format(feature) for feature in features])
195
196    stringify = lambda arr, name: ' {}="{}"'.format(name, ' '.join(arr)) if arr else ''
197    env_string = '{}{}{}{}{}{}'.format(
198            compiler,
199            features_str,
200            stringify(compiler_flags, 'COMPILER_FLAGS'),
201            stringify(configure_flags, 'CONFIGURE_FLAGS'),
202            stringify(get_extra_cflags(os, compiler), 'EXTRA_CFLAGS'),
203            extra_environment_vars)
204
205    job = '    - os: {}\n'.format(os)
206    job += '      arch: {}\n'.format(arch)
207    job += '      env: {}'.format(env_string)
208    return job
209
210
211def generate_unusual_combinations(unusuals, max_unusual_opts):
212    """
213    Generates different combinations of non-standard compilers, compiler flags,
214    configure flags and malloc_conf settings.
215
216    @param max_unusual_opts: Limit of unusual options per combination.
217    """
218    return chain.from_iterable(
219            [combinations(unusuals, i) for i in range(max_unusual_opts + 1)])
220
221
222def included(combination, exclude):
223    """
224    Checks if the combination of options should be included in the Travis
225    testing matrix.
226
227    @param exclude: A list of options to be avoided.
228    """
229    return not any(excluded in combination for excluded in exclude)
230
231
232def generate_jobs(os, arch, exclude, max_unusual_opts, unusuals=all_unusuals):
233    jobs = []
234    for combination in generate_unusual_combinations(unusuals, max_unusual_opts):
235        if included(combination, exclude):
236            jobs.append(format_job(os, arch, combination))
237    return '\n'.join(jobs)
238
239
240def generate_linux(arch):
241    os = LINUX
242
243    # Only generate 2 unusual options for AMD64 to reduce matrix size
244    max_unusual_opts = MAX_UNUSUAL_OPTIONS if arch == AMD64 else 1
245
246    exclude = []
247    if arch == PPC64LE:
248        # Avoid 32 bit builds and clang on PowerPC
249        exclude = (CROSS_COMPILE_32BIT, CLANG,)
250
251    return generate_jobs(os, arch, exclude, max_unusual_opts)
252
253
254def generate_macos(arch):
255    os = OSX
256
257    max_unusual_opts = 1
258
259    exclude = ([Option.as_malloc_conf(opt) for opt in (
260            'dss:primary',
261            'percpu_arena:percpu',
262            'background_thread:true')] +
263        [Option.as_configure_flag('--enable-prof')] +
264        [CLANG,])
265
266    return generate_jobs(os, arch, exclude, max_unusual_opts)
267
268
269def generate_windows(arch):
270    os = WINDOWS
271
272    max_unusual_opts = 3
273    unusuals = (
274        Option.as_configure_flag('--enable-debug'),
275        CL,
276        CROSS_COMPILE_32BIT,
277    )
278    return generate_jobs(os, arch, (), max_unusual_opts, unusuals)
279
280
281def generate_freebsd(arch):
282    os = FREEBSD
283
284    max_unusual_opts = 4
285    unusuals = (
286        Option.as_configure_flag('--enable-debug'),
287        Option.as_configure_flag('--enable-prof --enable-prof-libunwind'),
288        Option.as_configure_flag('--with-lg-page=16 --with-malloc-conf=tcache:false'),
289        CROSS_COMPILE_32BIT,
290    )
291    return generate_jobs(os, arch, (), max_unusual_opts, unusuals)
292
293
294
295def get_manual_jobs():
296    return """\
297    # Development build
298    - os: linux
299      env: CC=gcc CXX=g++ CONFIGURE_FLAGS="--enable-debug \
300--disable-cache-oblivious --enable-stats --enable-log --enable-prof" \
301EXTRA_CFLAGS="-Werror -Wno-array-bounds"
302    # --enable-expermental-smallocx:
303    - os: linux
304      env: CC=gcc CXX=g++ CONFIGURE_FLAGS="--enable-debug \
305--enable-experimental-smallocx --enable-stats --enable-prof" \
306EXTRA_CFLAGS="-Werror -Wno-array-bounds"
307"""
308
309
310def main():
311    jobs = '\n'.join((
312        generate_windows(AMD64),
313
314        generate_freebsd(AMD64),
315
316        generate_linux(AMD64),
317        generate_linux(PPC64LE),
318
319        generate_macos(AMD64),
320
321        get_manual_jobs(),
322    ))
323
324    print(TRAVIS_TEMPLATE.format(jobs=jobs))
325
326
327if __name__ == '__main__':
328    main()
329||||||| dec341af7695
330=======
331#!/usr/bin/env python
332
333from itertools import combinations
334
335travis_template = """\
336language: generic
337dist: precise
338
339matrix:
340  include:
341%s
342
343before_script:
344  - autoconf
345  - scripts/gen_travis.py > travis_script && diff .travis.yml travis_script
346  - ./configure ${COMPILER_FLAGS:+ \
347      CC="$CC $COMPILER_FLAGS" \
348      CXX="$CXX $COMPILER_FLAGS" } \
349      $CONFIGURE_FLAGS
350  - make -j3
351  - make -j3 tests
352
353script:
354  - make check
355"""
356
357# The 'default' configuration is gcc, on linux, with no compiler or configure
358# flags.  We also test with clang, -m32, --enable-debug, --enable-prof,
359# --disable-stats, and --with-malloc-conf=tcache:false.  To avoid abusing
360# travis though, we don't test all 2**7 = 128 possible combinations of these;
361# instead, we only test combinations of up to 2 'unusual' settings, under the
362# hope that bugs involving interactions of such settings are rare.
363# Things at once, for C(7, 0) + C(7, 1) + C(7, 2) = 29
364MAX_UNUSUAL_OPTIONS = 2
365
366os_default = 'linux'
367os_unusual = 'osx'
368
369compilers_default = 'CC=gcc CXX=g++'
370compilers_unusual = 'CC=clang CXX=clang++'
371
372compiler_flag_unusuals = ['-m32']
373
374configure_flag_unusuals = [
375    '--enable-debug',
376    '--enable-prof',
377    '--disable-stats',
378    '--disable-libdl',
379    '--enable-opt-safety-checks',
380]
381
382malloc_conf_unusuals = [
383    'tcache:false',
384    'dss:primary',
385    'percpu_arena:percpu',
386    'background_thread:true',
387]
388
389all_unusuals = (
390    [os_unusual] + [compilers_unusual] + compiler_flag_unusuals
391    + configure_flag_unusuals + malloc_conf_unusuals
392)
393
394unusual_combinations_to_test = []
395for i in xrange(MAX_UNUSUAL_OPTIONS + 1):
396    unusual_combinations_to_test += combinations(all_unusuals, i)
397
398gcc_multilib_set = False
399# Formats a job from a combination of flags
400def format_job(combination):
401    global gcc_multilib_set
402
403    os = os_unusual if os_unusual in combination else os_default
404    compilers = compilers_unusual if compilers_unusual in combination else compilers_default
405
406    compiler_flags = [x for x in combination if x in compiler_flag_unusuals]
407    configure_flags = [x for x in combination if x in configure_flag_unusuals]
408    malloc_conf = [x for x in combination if x in malloc_conf_unusuals]
409
410    # Filter out unsupported configurations on OS X.
411    if os == 'osx' and ('dss:primary' in malloc_conf or \
412      'percpu_arena:percpu' in malloc_conf or 'background_thread:true' \
413      in malloc_conf):
414        return ""
415    if len(malloc_conf) > 0:
416        configure_flags.append('--with-malloc-conf=' + ",".join(malloc_conf))
417
418    # Filter out an unsupported configuration - heap profiling on OS X.
419    if os == 'osx' and '--enable-prof' in configure_flags:
420        return ""
421
422    # We get some spurious errors when -Warray-bounds is enabled.
423    env_string = ('{} COMPILER_FLAGS="{}" CONFIGURE_FLAGS="{}" '
424	'EXTRA_CFLAGS="-Werror -Wno-array-bounds"').format(
425        compilers, " ".join(compiler_flags), " ".join(configure_flags))
426
427    job = ""
428    job += '    - os: %s\n' % os
429    job += '      env: %s\n' % env_string
430    if '-m32' in combination and os == 'linux':
431        job += '      addons:'
432        if gcc_multilib_set:
433            job += ' *gcc_multilib\n'
434        else:
435            job += ' &gcc_multilib\n'
436            job += '        apt:\n'
437            job += '          packages:\n'
438            job += '            - gcc-multilib\n'
439            gcc_multilib_set = True
440    return job
441
442include_rows = ""
443for combination in unusual_combinations_to_test:
444    include_rows += format_job(combination)
445
446# Development build
447include_rows += '''\
448    # Development build
449    - os: linux
450      env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-cache-oblivious --enable-stats --enable-log --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
451'''
452
453# Enable-expermental-smallocx
454include_rows += '''\
455    # --enable-expermental-smallocx:
456    - os: linux
457      env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds"
458'''
459
460# Valgrind build bots
461include_rows += '''
462    # Valgrind
463    - os: linux
464      env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" JEMALLOC_TEST_PREFIX="valgrind"
465      addons:
466        apt:
467          packages:
468            - valgrind
469'''
470
471# To enable valgrind on macosx add:
472#
473#  - os: osx
474#    env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" JEMALLOC_TEST_PREFIX="valgrind"
475#    install: brew install valgrind
476#
477# It currently fails due to: https://github.com/jemalloc/jemalloc/issues/1274
478
479print travis_template % include_rows
480>>>>>>> main
481