kunit_kernel.py (715a1284d89a740b197b3bad5eb20d36a397382f) kunit_kernel.py (09641f7c7d8f1309fe9ad9ce4e6a1697016d73ba)
1# SPDX-License-Identifier: GPL-2.0
2#
3# Runs UML kernel, collects output, and handles errors.
4#
5# Copyright (C) 2019, Google LLC.
6# Author: Felix Guo <felixguoxiuping@gmail.com>
7# Author: Brendan Higgins <brendanhiggins@google.com>
8
9import logging
10import subprocess
11import os
12import shutil
13import signal
1# SPDX-License-Identifier: GPL-2.0
2#
3# Runs UML kernel, collects output, and handles errors.
4#
5# Copyright (C) 2019, Google LLC.
6# Author: Felix Guo <felixguoxiuping@gmail.com>
7# Author: Brendan Higgins <brendanhiggins@google.com>
8
9import logging
10import subprocess
11import os
12import shutil
13import signal
14from typing import Iterator
14
15from contextlib import ExitStack
16
17import kunit_config
18import kunit_parser
19
20KCONFIG_PATH = '.config'
21KUNITCONFIG_PATH = '.kunitconfig'

--- 12 unchanged lines hidden (view full) ---

34
35class BuildError(Exception):
36 """Represents an error trying to build the Linux kernel."""
37
38
39class LinuxSourceTreeOperations(object):
40 """An abstraction over command line operations performed on a source tree."""
41
15
16from contextlib import ExitStack
17
18import kunit_config
19import kunit_parser
20
21KCONFIG_PATH = '.config'
22KUNITCONFIG_PATH = '.kunitconfig'

--- 12 unchanged lines hidden (view full) ---

35
36class BuildError(Exception):
37 """Represents an error trying to build the Linux kernel."""
38
39
40class LinuxSourceTreeOperations(object):
41 """An abstraction over command line operations performed on a source tree."""
42
42 def make_mrproper(self):
43 def make_mrproper(self) -> None:
43 try:
44 subprocess.check_output(['make', 'mrproper'], stderr=subprocess.STDOUT)
45 except OSError as e:
46 raise ConfigError('Could not call make command: ' + str(e))
47 except subprocess.CalledProcessError as e:
48 raise ConfigError(e.output.decode())
49
44 try:
45 subprocess.check_output(['make', 'mrproper'], stderr=subprocess.STDOUT)
46 except OSError as e:
47 raise ConfigError('Could not call make command: ' + str(e))
48 except subprocess.CalledProcessError as e:
49 raise ConfigError(e.output.decode())
50
50 def make_olddefconfig(self, build_dir, make_options):
51 def make_olddefconfig(self, build_dir, make_options) -> None:
51 command = ['make', 'ARCH=um', 'olddefconfig']
52 if make_options:
53 command.extend(make_options)
54 if build_dir:
55 command += ['O=' + build_dir]
56 try:
57 subprocess.check_output(command, stderr=subprocess.STDOUT)
58 except OSError as e:
59 raise ConfigError('Could not call make command: ' + str(e))
60 except subprocess.CalledProcessError as e:
61 raise ConfigError(e.output.decode())
62
52 command = ['make', 'ARCH=um', 'olddefconfig']
53 if make_options:
54 command.extend(make_options)
55 if build_dir:
56 command += ['O=' + build_dir]
57 try:
58 subprocess.check_output(command, stderr=subprocess.STDOUT)
59 except OSError as e:
60 raise ConfigError('Could not call make command: ' + str(e))
61 except subprocess.CalledProcessError as e:
62 raise ConfigError(e.output.decode())
63
63 def make_allyesconfig(self, build_dir, make_options):
64 def make_allyesconfig(self, build_dir, make_options) -> None:
64 kunit_parser.print_with_timestamp(
65 'Enabling all CONFIGs for UML...')
66 command = ['make', 'ARCH=um', 'allyesconfig']
67 if make_options:
68 command.extend(make_options)
69 if build_dir:
70 command += ['O=' + build_dir]
71 process = subprocess.Popen(

--- 5 unchanged lines hidden (view full) ---

77 'Disabling broken configs to run KUnit tests...')
78 with ExitStack() as es:
79 config = open(get_kconfig_path(build_dir), 'a')
80 disable = open(BROKEN_ALLCONFIG_PATH, 'r').read()
81 config.write(disable)
82 kunit_parser.print_with_timestamp(
83 'Starting Kernel with all configs takes a few minutes...')
84
65 kunit_parser.print_with_timestamp(
66 'Enabling all CONFIGs for UML...')
67 command = ['make', 'ARCH=um', 'allyesconfig']
68 if make_options:
69 command.extend(make_options)
70 if build_dir:
71 command += ['O=' + build_dir]
72 process = subprocess.Popen(

--- 5 unchanged lines hidden (view full) ---

78 'Disabling broken configs to run KUnit tests...')
79 with ExitStack() as es:
80 config = open(get_kconfig_path(build_dir), 'a')
81 disable = open(BROKEN_ALLCONFIG_PATH, 'r').read()
82 config.write(disable)
83 kunit_parser.print_with_timestamp(
84 'Starting Kernel with all configs takes a few minutes...')
85
85 def make(self, jobs, build_dir, make_options):
86 def make(self, jobs, build_dir, make_options) -> None:
86 command = ['make', 'ARCH=um', '--jobs=' + str(jobs)]
87 if make_options:
88 command.extend(make_options)
89 if build_dir:
90 command += ['O=' + build_dir]
91 try:
92 proc = subprocess.Popen(command,
93 stderr=subprocess.PIPE,
94 stdout=subprocess.DEVNULL)
95 except OSError as e:
96 raise BuildError('Could not call make command: ' + str(e))
97 _, stderr = proc.communicate()
98 if proc.returncode != 0:
99 raise BuildError(stderr.decode())
100 if stderr: # likely only due to build warnings
101 print(stderr.decode())
102
87 command = ['make', 'ARCH=um', '--jobs=' + str(jobs)]
88 if make_options:
89 command.extend(make_options)
90 if build_dir:
91 command += ['O=' + build_dir]
92 try:
93 proc = subprocess.Popen(command,
94 stderr=subprocess.PIPE,
95 stdout=subprocess.DEVNULL)
96 except OSError as e:
97 raise BuildError('Could not call make command: ' + str(e))
98 _, stderr = proc.communicate()
99 if proc.returncode != 0:
100 raise BuildError(stderr.decode())
101 if stderr: # likely only due to build warnings
102 print(stderr.decode())
103
103 def linux_bin(self, params, timeout, build_dir):
104 def linux_bin(self, params, timeout, build_dir) -> None:
104 """Runs the Linux UML binary. Must be named 'linux'."""
105 linux_bin = get_file_path(build_dir, 'linux')
106 outfile = get_outfile_path(build_dir)
107 with open(outfile, 'w') as output:
108 process = subprocess.Popen([linux_bin] + params,
109 stdout=output,
110 stderr=subprocess.STDOUT)
111 process.wait(timeout)
112
105 """Runs the Linux UML binary. Must be named 'linux'."""
106 linux_bin = get_file_path(build_dir, 'linux')
107 outfile = get_outfile_path(build_dir)
108 with open(outfile, 'w') as output:
109 process = subprocess.Popen([linux_bin] + params,
110 stdout=output,
111 stderr=subprocess.STDOUT)
112 process.wait(timeout)
113
113def get_kconfig_path(build_dir):
114def get_kconfig_path(build_dir) -> str:
114 return get_file_path(build_dir, KCONFIG_PATH)
115
115 return get_file_path(build_dir, KCONFIG_PATH)
116
116def get_kunitconfig_path(build_dir):
117def get_kunitconfig_path(build_dir) -> str:
117 return get_file_path(build_dir, KUNITCONFIG_PATH)
118
118 return get_file_path(build_dir, KUNITCONFIG_PATH)
119
119def get_outfile_path(build_dir):
120def get_outfile_path(build_dir) -> str:
120 return get_file_path(build_dir, OUTFILE_PATH)
121
122class LinuxSourceTree(object):
123 """Represents a Linux kernel source tree with KUnit tests."""
124
121 return get_file_path(build_dir, OUTFILE_PATH)
122
123class LinuxSourceTree(object):
124 """Represents a Linux kernel source tree with KUnit tests."""
125
125 def __init__(self):
126 def __init__(self) -> None:
126 self._ops = LinuxSourceTreeOperations()
127 signal.signal(signal.SIGINT, self.signal_handler)
128
127 self._ops = LinuxSourceTreeOperations()
128 signal.signal(signal.SIGINT, self.signal_handler)
129
129 def clean(self):
130 def clean(self) -> bool:
130 try:
131 self._ops.make_mrproper()
132 except ConfigError as e:
133 logging.error(e)
134 return False
135 return True
136
131 try:
132 self._ops.make_mrproper()
133 except ConfigError as e:
134 logging.error(e)
135 return False
136 return True
137
137 def create_kunitconfig(self, build_dir, defconfig=DEFAULT_KUNITCONFIG_PATH):
138 def create_kunitconfig(self, build_dir, defconfig=DEFAULT_KUNITCONFIG_PATH) -> None:
138 kunitconfig_path = get_kunitconfig_path(build_dir)
139 if not os.path.exists(kunitconfig_path):
140 shutil.copyfile(defconfig, kunitconfig_path)
141
139 kunitconfig_path = get_kunitconfig_path(build_dir)
140 if not os.path.exists(kunitconfig_path):
141 shutil.copyfile(defconfig, kunitconfig_path)
142
142 def read_kunitconfig(self, build_dir):
143 def read_kunitconfig(self, build_dir) -> None:
143 kunitconfig_path = get_kunitconfig_path(build_dir)
144 self._kconfig = kunit_config.Kconfig()
145 self._kconfig.read_from_file(kunitconfig_path)
146
144 kunitconfig_path = get_kunitconfig_path(build_dir)
145 self._kconfig = kunit_config.Kconfig()
146 self._kconfig.read_from_file(kunitconfig_path)
147
147 def validate_config(self, build_dir):
148 def validate_config(self, build_dir) -> bool:
148 kconfig_path = get_kconfig_path(build_dir)
149 validated_kconfig = kunit_config.Kconfig()
150 validated_kconfig.read_from_file(kconfig_path)
151 if not self._kconfig.is_subset_of(validated_kconfig):
152 invalid = self._kconfig.entries() - validated_kconfig.entries()
153 message = 'Provided Kconfig is not contained in validated .config. Following fields found in kunitconfig, ' \
154 'but not in .config: %s' % (
155 ', '.join([str(e) for e in invalid])
156 )
157 logging.error(message)
158 return False
159 return True
160
149 kconfig_path = get_kconfig_path(build_dir)
150 validated_kconfig = kunit_config.Kconfig()
151 validated_kconfig.read_from_file(kconfig_path)
152 if not self._kconfig.is_subset_of(validated_kconfig):
153 invalid = self._kconfig.entries() - validated_kconfig.entries()
154 message = 'Provided Kconfig is not contained in validated .config. Following fields found in kunitconfig, ' \
155 'but not in .config: %s' % (
156 ', '.join([str(e) for e in invalid])
157 )
158 logging.error(message)
159 return False
160 return True
161
161 def build_config(self, build_dir, make_options):
162 def build_config(self, build_dir, make_options) -> bool:
162 kconfig_path = get_kconfig_path(build_dir)
163 if build_dir and not os.path.exists(build_dir):
164 os.mkdir(build_dir)
165 self._kconfig.write_to_file(kconfig_path)
166 try:
167 self._ops.make_olddefconfig(build_dir, make_options)
168 except ConfigError as e:
169 logging.error(e)
170 return False
171 return self.validate_config(build_dir)
172
163 kconfig_path = get_kconfig_path(build_dir)
164 if build_dir and not os.path.exists(build_dir):
165 os.mkdir(build_dir)
166 self._kconfig.write_to_file(kconfig_path)
167 try:
168 self._ops.make_olddefconfig(build_dir, make_options)
169 except ConfigError as e:
170 logging.error(e)
171 return False
172 return self.validate_config(build_dir)
173
173 def build_reconfig(self, build_dir, make_options):
174 def build_reconfig(self, build_dir, make_options) -> bool:
174 """Creates a new .config if it is not a subset of the .kunitconfig."""
175 kconfig_path = get_kconfig_path(build_dir)
176 if os.path.exists(kconfig_path):
177 existing_kconfig = kunit_config.Kconfig()
178 existing_kconfig.read_from_file(kconfig_path)
179 if not self._kconfig.is_subset_of(existing_kconfig):
180 print('Regenerating .config ...')
181 os.remove(kconfig_path)
182 return self.build_config(build_dir, make_options)
183 else:
184 return True
185 else:
186 print('Generating .config ...')
187 return self.build_config(build_dir, make_options)
188
175 """Creates a new .config if it is not a subset of the .kunitconfig."""
176 kconfig_path = get_kconfig_path(build_dir)
177 if os.path.exists(kconfig_path):
178 existing_kconfig = kunit_config.Kconfig()
179 existing_kconfig.read_from_file(kconfig_path)
180 if not self._kconfig.is_subset_of(existing_kconfig):
181 print('Regenerating .config ...')
182 os.remove(kconfig_path)
183 return self.build_config(build_dir, make_options)
184 else:
185 return True
186 else:
187 print('Generating .config ...')
188 return self.build_config(build_dir, make_options)
189
189 def build_um_kernel(self, alltests, jobs, build_dir, make_options):
190 def build_um_kernel(self, alltests, jobs, build_dir, make_options) -> bool:
190 try:
191 if alltests:
192 self._ops.make_allyesconfig(build_dir, make_options)
193 self._ops.make_olddefconfig(build_dir, make_options)
194 self._ops.make(jobs, build_dir, make_options)
195 except (ConfigError, BuildError) as e:
196 logging.error(e)
197 return False
198 return self.validate_config(build_dir)
199
191 try:
192 if alltests:
193 self._ops.make_allyesconfig(build_dir, make_options)
194 self._ops.make_olddefconfig(build_dir, make_options)
195 self._ops.make(jobs, build_dir, make_options)
196 except (ConfigError, BuildError) as e:
197 logging.error(e)
198 return False
199 return self.validate_config(build_dir)
200
200 def run_kernel(self, args=[], build_dir='', timeout=None):
201 args.extend(['mem=1G'])
201 def run_kernel(self, args=[], build_dir='', timeout=None) -> Iterator[str]:
202 args.extend(['mem=1G', 'console=tty'])
202 self._ops.linux_bin(args, timeout, build_dir)
203 outfile = get_outfile_path(build_dir)
204 subprocess.call(['stty', 'sane'])
205 with open(outfile, 'r') as file:
206 for line in file:
207 yield line
208
203 self._ops.linux_bin(args, timeout, build_dir)
204 outfile = get_outfile_path(build_dir)
205 subprocess.call(['stty', 'sane'])
206 with open(outfile, 'r') as file:
207 for line in file:
208 yield line
209
209 def signal_handler(self, sig, frame):
210 def signal_handler(self, sig, frame) -> None:
210 logging.error('Build interruption occurred. Cleaning console.')
211 subprocess.call(['stty', 'sane'])
211 logging.error('Build interruption occurred. Cleaning console.')
212 subprocess.call(['stty', 'sane'])