From c8d64826e35cc6be399e7cf50bbf4f393d3e9a00 Mon Sep 17 00:00:00 2001 From: Jan Philipp Hafer Date: Fri, 6 Dec 2024 21:58:45 +0100 Subject: [PATCH 1/2] remove trailing lines --- cpuset/commands/proc.py | 64 +++++++++++++++++++-------------------- cpuset/commands/set.py | 18 +++++------ cpuset/commands/shield.py | 28 ++++++++--------- cpuset/config.py | 4 +-- cpuset/cset.py | 50 +++++++++++++++--------------- cpuset/main.py | 10 +++--- cpuset/util.py | 6 ++-- 7 files changed, 90 insertions(+), 90 deletions(-) diff --git a/cpuset/commands/proc.py b/cpuset/commands/proc.py index 4eb2f22..5fb60ff 100644 --- a/cpuset/commands/proc.py +++ b/cpuset/commands/proc.py @@ -27,10 +27,10 @@ from cpuset import cset from cpuset.util import * from cpuset.commands.common import * -try: from cpuset.commands import set +try: from cpuset.commands import set except SyntaxError: raise -except: +except: pass global log @@ -57,7 +57,7 @@ and specifying the cpuset that the process is to be created in. For example: - # cset proc --set=blazing_cpuset --exec /usr/bin/fast_code + # cset proc --set=blazing_cpuset --exec /usr/bin/fast_code This command will execute the /usr/bin/fast_code program on the "blazing_cpuset" cpuset. @@ -126,7 +126,7 @@ kernel threads will be added to the move. Unbound kernel threads are those that can run on any CPU. If you also specify the --force switch, then all tasks, kernel or not, bound or not, -will be moved. +will be moved. CAUTION: Please be cautious with the --force switch, since moving a kernel thread that is bound to a specific CPU to a cpuset that @@ -207,14 +207,14 @@ def func(parser, options, args): cset.rescan() - tset = None + tset = None if options.list or options.exc: if options.set: tset = cset.unique_set(options.set) elif options.toset: tset = cset.unique_set(options.toset) elif len(args) > 0: - if options.exc: + if options.exc: tset = cset.unique_set(args[0]) del args[0] else: @@ -270,7 +270,7 @@ def func(parser, options, args): set.active(tset) # next, if there is a pidspec, move just that if options.pid: - if options.fromset and not options.force: + if options.fromset and not options.force: fset = cset.unique_set(options.fromset) elif options.toset and options.set: fset = cset.unique_set(options.set) @@ -295,15 +295,15 @@ def func(parser, options, args): raise CpusetException("origination cpuset not specified") nt = len(fset.tasks) if nt == 0: - raise CpusetException('no tasks to move from cpuset "%s"' + raise CpusetException('no tasks to move from cpuset "%s"' % fset.path) if options.move: - log.info('moving all tasks from %s to %s', + log.info('moving all tasks from %s to %s', fset.name, tset.path) selective_move(fset, tset, None, options.kthread, options.force, options.threads) else: - log.info('moving all kernel threads from %s to %s', + log.info('moving all kernel threads from %s to %s', fset.path, tset.path) # this is a -k "move", so only move kernel threads pids = [] @@ -321,12 +321,12 @@ def list_sets(args): log.debug("entering list_sets, args=%s", args) l = [] if isinstance(args, list): - for s in args: + for s in args: if isstr(s): l.extend(cset.find_sets(s)) elif not isinstance(s, cset.CpuSet): raise CpusetException( - 'list_sets() args=%s, of which "%s" not a string or CpuSet' + 'list_sets() args=%s, of which "%s" not a string or CpuSet' % (args, s)) else: l.append(s) @@ -350,7 +350,7 @@ def list_sets(args): log.info(cset.summary(s)) def move(fromset, toset, plist=None, verb=None, force=None): - log.debug('entering move, fromset=%s toset=%s list=%s force=%s verb=%s', + log.debug('entering move, fromset=%s toset=%s list=%s force=%s verb=%s', fromset, toset, plist, force, verb) if isstr(fromset): fset = cset.unique_set(fromset) @@ -368,15 +368,15 @@ def move(fromset, toset, plist=None, verb=None, force=None): tset = toset if plist == None: log.debug('moving default of all processes') - if tset != fset and not force: + if tset != fset and not force: plist = fset.tasks else: raise CpusetException( "cannot move tasks into their origination cpuset") output = 0 - if verb: + if verb: output = verb - elif verbose: + elif verbose: output = verbose if output: l = [] @@ -410,7 +410,7 @@ def selective_move(fset, tset, plist=None, kthread=None, force=None, threads=Non ktskb = 0 sstsk = 0 target = cset.unique_set(tset) - if fset: + if fset: fset = cset.unique_set(fset) if fset == target and not force: raise CpusetException( @@ -426,7 +426,7 @@ def selective_move(fset, tset, plist=None, kthread=None, force=None, threads=Non # kernel threads do not have an excutable image os.readlink('/proc/'+task+'/exe') autsk += 1 - if fset and not force: + if fset and not force: try: task_check.index(task) tasks.append(task) @@ -440,9 +440,9 @@ def selective_move(fset, tset, plist=None, kthread=None, force=None, threads=Non if thread != task: log.debug(' adding thread %s', thread) tasks.append(thread) - utsk += 1 + utsk += 1 except ValueError: - log.debug(' task %s not running in %s, skipped', + log.debug(' task %s not running in %s, skipped', task, fset.name) utsknr += 1 else: @@ -459,11 +459,11 @@ def selective_move(fset, tset, plist=None, kthread=None, force=None, threads=Non # this is in try because the task may not exist by the # time we do this, in that case, just ignore it if kthread: - if force: + if force: tasks.append(task) ktsk += 1 else: - if is_unbound(task): + if is_unbound(task): tasks.append(task) ktsk += 1 elif cset.lookup_task_from_cpusets(task) == target.path: @@ -480,7 +480,7 @@ def selective_move(fset, tset, plist=None, kthread=None, force=None, threads=Non ktsknr += 1 # ok, move 'em log.debug('moving %d tasks to %s ...', len(tasks), tset.name) - if len(tasks) == 0: + if len(tasks) == 0: log.info('**> no task matched move criteria') if sstsk > 0: raise CpusetException('same source/destination cpuset, use --force if ok') @@ -535,8 +535,8 @@ def run(tset, args, usr_par=None, grp_par=None): log.debug('entering run, set=%s args=%s ', s.path, args) set.active(s) # check user - if usr_par: - try: + if usr_par: + try: user = pwd.getpwnam(usr_par)[2] except KeyError: try: @@ -562,11 +562,11 @@ def run(tset, args, usr_par=None, grp_par=None): pass # just forget it # move myself into target cpuset and exec child move_pidspec(str(os.getpid()), s) - log.info('--> last message, executed args into cpuset "%s", new pid is: %s', - s.path, os.getpid()) + log.info('--> last message, executed args into cpuset "%s", new pid is: %s', + s.path, os.getpid()) # change user and group before exec if grp_par: os.setgid(group) - if usr_par: + if usr_par: os.setuid(user) os.environ["LOGNAME"] = usr_par os.environ["USERNAME"] = usr_par @@ -574,22 +574,22 @@ def run(tset, args, usr_par=None, grp_par=None): os.execvp(args[0], args) def is_unbound(proc): - # FIXME: popen is slow... + # FIXME: popen is slow... # --> use /proc//status -> Cpus_allowed # int(line.replace(',',''), 16) # note: delete leading zeros to compare to allcpumask line = os.popen('/usr/bin/taskset -p ' + str(proc) +' 2>/dev/null', 'r').readline() aff = line.split()[-1] - log.debug('is_unbound, proc=%s aff=%s allcpumask=%s', + log.debug('is_unbound, proc=%s aff=%s allcpumask=%s', proc, aff, cset.allcpumask) if aff == cset.allcpumask: return True return False def pidspec_to_list(pidspec, fset=None, threads=False): """create a list of process ids out of a pidspec""" - log.debug('entering pidspecToList, pidspec=%s fset=%s threads=%s', + log.debug('entering pidspecToList, pidspec=%s fset=%s threads=%s', pidspec, fset, threads) - if fset: + if fset: if isstr(fset): fset = cset.unique_set(fset) elif not isinstance(fset, cset.CpuSet): raise CpusetException("passed fset=%s, which is not a string or CpuSet" % fset) diff --git a/cpuset/commands/set.py b/cpuset/commands/set.py index e7ae61e..5b57ea8 100644 --- a/cpuset/commands/set.py +++ b/cpuset/commands/set.py @@ -30,7 +30,7 @@ try: from cpuset.commands import proc except SyntaxError: raise -except: +except: pass global log @@ -151,7 +151,7 @@ help = 'do things recursively, use with --list and --destroy', action = 'store_true'), make_option('--force', - help = 'force recursive deletion even if processes are running ' + help = 'force recursive deletion even if processes are running ' 'in those cpusets (they will be moved to parent cpusets)', action = 'store_true'), make_option('-x', '--usehex', @@ -254,7 +254,7 @@ def destroy_sets(sets, recurse=False, force=False): nl.append(sets) # check that sets passed are ok, will raise if one is bad sl2 = [] - for s in nl: + for s in nl: st = cset.unique_set(s) sl2.append(st) if len(st.subsets) > 0: @@ -291,7 +291,7 @@ def destroy(name): set = cset.unique_set(name) elif not isinstance(name, cset.CpuSet): raise CpusetException( - "passed name=%s, which is not a string or CpuSet" % name) + "passed name=%s, which is not a string or CpuSet" % name) else: set = name tsks = set.tasks @@ -361,7 +361,7 @@ def create_from_options(options, args): mspec = None cx = None mx = None - if options.cpu: + if options.cpu: cset.cpuspec_check(options.cpu) cspec = options.cpu if options.mem: @@ -385,11 +385,11 @@ def create(name, cpuspec, memspec, cx, mx): try: cset.unique_set(name) except CpusetNotFound: - pass + pass except: raise CpusetException('cpuset "%s" not unique, please specify by path' % name) else: - raise CpusetExists('attempt to create already existing set: "%s"' % name) + raise CpusetExists('attempt to create already existing set: "%s"' % name) # FIXME: check if name is a path here os.mkdir(cset.CpuSet.basepath+'/'+name) # fixme: perhaps reparsing the all the sets is not so efficient... @@ -405,7 +405,7 @@ def modify(name, cpuspec=None, memspec=None, cx=None, mx=None): nset = cset.unique_set(name) elif not isinstance(name, cset.CpuSet): raise CpusetException( - "passed name=%s, which is not a string or CpuSet" % name) + "passed name=%s, which is not a string or CpuSet" % name) else: nset = name log.debug('modifying cpuset "%s"', nset.name) @@ -472,7 +472,7 @@ def set_details(name, indent=None, width=None, usehex=False): if config.mread: l.append(set.path) l2 = [] - for line in l: + for line in l: l2.append(line.strip()) return ';'.join(l2) diff --git a/cpuset/commands/shield.py b/cpuset/commands/shield.py index 7148111..1ca322d 100644 --- a/cpuset/commands/shield.py +++ b/cpuset/commands/shield.py @@ -30,7 +30,7 @@ from cpuset.util import * from cpuset import config -global log +global log log = logging.getLogger('shield') help = 'supercommand to set up and manage basic shielding' @@ -143,14 +143,14 @@ You can adjust which CPUs are in the shielded cpuset by issuing the --cpu subcommand again anytime after the shield has been -initialized. +initialized. For example if the original shield contained CPU0 and CPU1 in the system set and CPU2 and CPU3 in the user set, if you then issue the following command: - + # cset shield --cpu=1,2,3 - + then that command will move CPU1 into the shielded "user" cpuset. Any processes or threads that were running on CPU1 that belonged to the unshielded "system" cpuset are migrated to CPU0 by the @@ -225,10 +225,10 @@ def func(parser, options, args): if options.verbose: verbose = options.verbose cset.rescan() - if options.sysset: + if options.sysset: global SYS_SET SYS_SET = options.sysset - if options.userset: + if options.userset: global USR_SET USR_SET = options.userset @@ -254,16 +254,16 @@ def func(parser, options, args): options.shield = True else: return - - if options.reset: + + if options.reset: reset_shield() return - if options.cpu: + if options.cpu: make_shield(options.cpu, options.kthread) return - if options.kthread: + if options.kthread: make_kthread(options.kthread) return @@ -271,7 +271,7 @@ def func(parser, options, args): exec_args(args, options.user, options.group) # exec_args does not return... - if options.shield or options.unshield: + if options.shield or options.unshield: shield_exists() if options.shield: smsg = 'shielding' @@ -351,11 +351,11 @@ def reset_shield(): log.info("--> deactivating/reseting shielding") shield_exists() tasks = cset.unique_set(USR_SET).tasks - log.info('moving %s tasks from "%s" user set to root set...', + log.info('moving %s tasks from "%s" user set to root set...', len(tasks), USR_SET) proc.move(USR_SET, 'root', None, verbose) tasks = cset.unique_set(SYS_SET).tasks - log.info('moving %s tasks from "%s" system set to root set...', + log.info('moving %s tasks from "%s" system set to root set...', len(tasks), SYS_SET) proc.move(SYS_SET, 'root', None, verbose) log.info('deleting "%s" and "%s" sets', USR_SET, SYS_SET) @@ -434,7 +434,7 @@ def make_kthread(state): if state == 'on': log.info('--> activating kthread shielding') root_tasks = cset.unique_set('/').tasks - log.debug('root set has %d tasks, checking for unbound', + log.debug('root set has %d tasks, checking for unbound', len(root_tasks)) tasks = [] for task in root_tasks: diff --git a/cpuset/config.py b/cpuset/config.py index 76edc45..b94353f 100644 --- a/cpuset/config.py +++ b/cpuset/config.py @@ -19,7 +19,7 @@ This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License @@ -36,7 +36,7 @@ ############################################################################ defloc = '/etc/cset.conf' # default config file location mread = False # machine readable output, usually set - # via option -m/--machine + # via option -m/--machine mountpoint = '/cpusets' # cpuset filessytem mount point ############################################################################ diff --git a/cpuset/cset.py b/cpuset/cset.py index a73d44b..250c17c 100644 --- a/cpuset/cset.py +++ b/cpuset/cset.py @@ -14,7 +14,7 @@ This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License @@ -24,7 +24,7 @@ import os, re, sys, logging -if __name__ == '__main__': +if __name__ == '__main__': sys.path.insert(0, "..") logging.basicConfig() @@ -33,7 +33,7 @@ RootSet = None class CpuSet(object): - # sets is a class variable dict that keeps track of all + # sets is a class variable dict that keeps track of all # cpusets discovered such that we can link them in properly. # The basepath is it's base path, the sets are indexed via # a relative path from this basepath. @@ -60,7 +60,7 @@ def __init__(self, path=None): self.name = 'root' self.path = '/' self.parent = self - if (CpuSet.sets): + if (CpuSet.sets): del CpuSet.sets CpuSet.sets = {} CpuSet.sets[self.path] = self @@ -82,7 +82,7 @@ def __init__(self, path=None): relpath = (os.path.join(dir,sub).replace(CpuSet.basepath, '') if len(sub) > 0 else '/') node.subsets.append(CpuSet.sets[relpath]) - log.debug("%s has %i subsets: [%s]", dir, + log.debug("%s has %i subsets: [%s]", dir, len(node.subsets), '|'.join(dirs)) log.debug("staring top-down parenting walk...") @@ -174,32 +174,32 @@ def write_01_to(self, file_to_write, value): def delprop(self): raise AttributeError("deletion of properties not allowed") - def getcpus(self): + def getcpus(self): return self.read_first_line_from(CpuSet.cpus_path) def setcpus(self, newval): cpuspec_check(newval) self.write_value_to(CpuSet.cpus_path, newval) cpus = property(fget=getcpus, fset=setcpus, fdel=delprop, doc="CPU specifier") - def getmems(self): + def getmems(self): return self.read_first_line_from(CpuSet.mems_path) - def setmems(self, newval): + def setmems(self, newval): # FIXME: check format for correctness self.write_value_to(CpuSet.mems_path, newval) mems = property(getmems, setmems, delprop, "Mem node specifier") - - def getcpuxlsv(self): + + def getcpuxlsv(self): return self.read_first_line_from(CpuSet.cpu_exclusive_path) == '1' def setcpuxlsv(self, newval): self.write_01_to(CpuSet.cpu_exclusive_path, newval) - cpu_exclusive = property(getcpuxlsv, setcpuxlsv, delprop, + cpu_exclusive = property(getcpuxlsv, setcpuxlsv, delprop, "CPU exclusive flag") - def getmemxlsv(self): + def getmemxlsv(self): return self.read_first_line_from(CpuSet.mem_exclusive_path) == '1' def setmemxlsv(self, newval): self.write_01_to(CpuSet.mem_exclusive_path, newval) - mem_exclusive = property(getmemxlsv, setmemxlsv, delprop, + mem_exclusive = property(getmemxlsv, setmemxlsv, delprop, "Memory exclusive flag") def gettasks(self): @@ -226,7 +226,7 @@ def settasks(self, tasklist): notfound.append(task) elif str(err).find('Invalid argument'): unmovable.append(task) - else: + else: raise if prog: tick += 1 @@ -237,8 +237,8 @@ def settasks(self, tasklist): if len(unmovable) > 0: log.info('**> %s tasks are not movable, impossible to move', len(unmovable)) log.debug(' not movable: %s', unmovable) - log.debug("-> prop_set %s.tasks set with %s tasks", self.path, - len(tasklist)) + log.debug("-> prop_set %s.tasks set with %s tasks", self.path, + len(tasklist)) tasks = property(gettasks, settasks, delprop, "Task list") # @@ -284,7 +284,7 @@ def unique_set(name): raise CpusetException('unique_set() passed None as arg') if isinstance(name, CpuSet): return name nl = find_sets(name) - if len(nl) > 1: + if len(nl) > 1: raise CpusetNotUnique('cpuset name "%s" not unique: %s' % (name, [x.path for x in nl]) ) return nl[0] @@ -324,9 +324,9 @@ def walk_set(set): yield node for node in set.subsets: - for result in walk_set(node): - log.debug("++++++ yield %s", node.name) - yield result + for result in walk_set(node): + log.debug("++++++ yield %s", node.name) + yield result def rescan(): """re-read the cpuset directory to sync system with data structs""" @@ -378,7 +378,7 @@ def cpuspec_to_hex(cpuspec): # one cpu in this group log.debug(" adding cpu %s to result", items[0]) number |= 1 << int(items[0]) - elif len(items) == 2: + elif len(items) == 2: il = [int(ii) for ii in items] if il[1] >= il[0]: rng = list(range(il[0], il[1]+1)) else: rng = list(range(il[1], il[0]+1)) @@ -405,7 +405,7 @@ def cpuspec_inverse(cpuspec): """calculate inverse of cpu specification""" cpus = [0 for x in range(maxcpu+1)] groups = cpuspec.split(',') - log.debug("cpuspec_inverse(%s) maxcpu=%d groups=%d", + log.debug("cpuspec_inverse(%s) maxcpu=%d groups=%d", cpuspec, maxcpu, len(groups)) for set in groups: items = set.split('-') @@ -431,11 +431,11 @@ def cpuspec_inverse(cpuspec): for x in range(0, len(cpus)): if cpus[x] == 0 and ingrp: nspec += str(begin) - if x > begin+1: + if x > begin+1: nspec += '-' + str(x if cpus[x] else x-1) ingrp = False if cpus[x] == 1: - if not ingrp: + if not ingrp: if len(nspec): nspec += ',' begin = x ingrp = True @@ -453,7 +453,7 @@ def summary(set): else: msg = 'tasks' return ('"%s" cpuset of CPUSPEC(%s) with %s %s running' % (set.name, set.cpus, len(set.tasks), msg) ) - + def calc_cpumask(max): all = 1 ii = 1 diff --git a/cpuset/main.py b/cpuset/main.py index f86f61d..0d92669 100644 --- a/cpuset/main.py +++ b/cpuset/main.py @@ -49,14 +49,14 @@ def canonical_cmd(self, key): sys.exit(1) return candidates[0] - + def __getitem__(self, key): """Return the command python module name based. """ global prog cmd_mod = self.get(key) or self.get(self.canonical_cmd(key)) - + __import__('cpuset.commands.' + cmd_mod) return getattr(cpuset.commands, cmd_mod) @@ -73,7 +73,7 @@ def __getitem__(self, key): def _print_helpstring(cmd): print(' ' + cmd + ' ' * (12 - len(cmd)) + commands[cmd].help) - + def print_help(): print('Usage: %s [global options] [command options]' % os.path.basename(sys.argv[0])) print() @@ -89,7 +89,7 @@ def print_help(): cmds = list(commands.keys()) cmds.sort() - print() + print() print('Super commands (high-level and multi-function):') for cmd in supercommands: _print_helpstring(cmd) @@ -115,7 +115,7 @@ def main(): print('usage: %s ' % prog, file=sys.stderr) print(' Try "%s --help" for a list of supported commands' % prog, file=sys.stderr) sys.exit(1) - + # configure logging import logging console = logging.StreamHandler(sys.stdout) diff --git a/cpuset/util.py b/cpuset/util.py index 16cd0bd..18e97ca 100644 --- a/cpuset/util.py +++ b/cpuset/util.py @@ -53,11 +53,11 @@ def __init__(self, finalcount, progresschar=None): self.blockcount=0 self.finished=False # Use dark shade (U+2593) char for progress if none passed - if not progresschar: + if not progresschar: self.block='\u2593' - else: + else: self.block=progresschar - if config.mread: + if config.mread: return self.f=sys.stdout if not self.finalcount: return From 6f30ca9bcc4cbb8a3928d2b76745fac623d09b1e Mon Sep 17 00:00:00 2001 From: Jan Philipp Hafer Date: Fri, 6 Dec 2024 22:11:22 +0100 Subject: [PATCH 2/2] Improve process spawn speed. Problem: # FIXME: popen is slow Solution: use subprocess.run --- cpuset/commands/proc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cpuset/commands/proc.py b/cpuset/commands/proc.py index 5fb60ff..4f274fe 100644 --- a/cpuset/commands/proc.py +++ b/cpuset/commands/proc.py @@ -21,6 +21,7 @@ """ import sys, os, io, re, logging, pwd, grp +import subprocess from optparse import OptionParser, make_option from cpuset import config @@ -574,11 +575,11 @@ def run(tset, args, usr_par=None, grp_par=None): os.execvp(args[0], args) def is_unbound(proc): - # FIXME: popen is slow... # --> use /proc//status -> Cpus_allowed # int(line.replace(',',''), 16) # note: delete leading zeros to compare to allcpumask - line = os.popen('/usr/bin/taskset -p ' + str(proc) +' 2>/dev/null', 'r').readline() + taskset_res = subprocess.run(['/usr/bin/taskset -p', str(proc)], capture_output=True) + line = taskset_res.stdout.readline() aff = line.split()[-1] log.debug('is_unbound, proc=%s aff=%s allcpumask=%s', proc, aff, cset.allcpumask)