Skip to content

add support for setting environment variables via 'pushenv' with modextravars #4030

New issue

Have a question about this project? Sign up for a free account to open an issue and contact its maintainers and the community.

Sign up for

By clicking “Sign up for ”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on ? Sign in to your account

Merged
merged 3 commits into from
Jul 5, 2022
Merged

add support for setting environment variables via 'pushenv' with modextravars #4030

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8d91b7a
add support for setting environment variables via 'pushenv' with mode…
boegelJun 13, 2022
c1cf8b0
bump to Lmod 8.7.6 for running tests (which includes a fix for pushen…
boegelJun 28, 2022
167f30f
avoid using '-' in name of environment variable in Toy_Extension easy…
boegelJul 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ./workflows/bootstrap_script.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@ jobs:
runs-on: ubuntu-latest
outputs:
lmod7: Lmod-7.8.22
lmod8: Lmod-8.7.4
lmod8: Lmod-8.7.6
modulesTcl: modules-tcl-1.147
modules3: modules-3.2.10
modules4: modules-4.1.4
Expand Down
2 changes: 1 addition & 1 deletion ./workflows/container_tests.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -39,7 +39,7 @@ jobs:
cd $HOME
export INSTALL_DEP=$_WORKSPACE/easybuild/scripts/install_eb_dep.sh
# install Lmod
source $INSTALL_DEP Lmod-8.7.4 $HOME
source $INSTALL_DEP Lmod-8.7.6 $HOME
# changes in environment are not passed to other steps, so need to create files...
echo $MOD_INIT > mod_init
echo $PATH > path
Expand Down
2 changes: 1 addition & 1 deletion ./workflows/eb_command.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,7 +38,7 @@ jobs:
cd $HOME
export INSTALL_DEP=$_WORKSPACE/easybuild/scripts/install_eb_dep.sh
# install Lmod
source $INSTALL_DEP Lmod-8.7.4 $HOME
source $INSTALL_DEP Lmod-8.7.6 $HOME
# changes in environment are not passed to other steps, so need to create files...
echo $MOD_INIT > mod_init
echo $PATH > path
Expand Down
2 changes: 1 addition & 1 deletion ./workflows/unit_tests.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,7 +6,7 @@ jobs:
runs-on: ubuntu-latest
outputs:
lmod7: Lmod-7.8.22
lmod8: Lmod-8.7.4
lmod8: Lmod-8.7.6
modulesTcl: modules-tcl-1.147
modules3: modules-3.2.10
modules4: modules-4.1.4
Expand Down
36 changes: 34 additions & 2 deletions easybuild/tools/module_generator.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -407,6 +407,29 @@ def det_installdir(self, modfile):

return res

def unpack_setenv_value(self, env_var_name, env_var_val):
"""
Unpack value that specifies how to define an environment variable with specified name.
"""
use_pushenv = False

# value may be specified as a string, or as a dict for special cases
if isinstance(env_var_val, string_type):
value = env_var_val

elif isinstance(env_var_val, dict):
use_pushenv = env_var_val.get('pushenv', False)
try:
value = env_var_val['value']
except KeyError:
raise EasyBuildError("Required key 'value' is missing in dict that specifies how to set $%s: %s",
env_var_name, env_var_val)
else:
raise EasyBuildError("Incorrect value type for setting $%s environment variable (%s): %s",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this include list of valid types?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, that would be helpful indeed, I'll change that, thanks for the suggestion!

env_var_name, type(env_var_val), env_var_val)

return value, use_pushenv

# From this point on just not implemented methods

def check_group(self, group, error_msg=None):
Expand DownExpand Up@@ -1019,6 +1042,8 @@ def set_environment(self, key, value, relpath=False):
self.log.info("Not including statement to define environment variable $%s, as specified", key)
return ''

value, use_pushenv = self.unpack_setenv_value(key, value)

# quotes are needed, to ensure smooth working of EBDEVEL* modulefiles
if relpath:
if value:
Expand All@@ -1027,7 +1052,9 @@ def set_environment(self, key, value, relpath=False):
val = '"$root"'
else:
val = quote_str(value, tcl=True)
return 'setenv\t%s\t\t%s\n' % (key, val)

env_setter = 'pushenv' if use_pushenv else 'setenv'
return '%s\t%s\t\t%s\n' % (env_setter, key, val)

def swap_module(self, mod_name_out, mod_name_in, guarded=True):
"""
Expand DownExpand Up@@ -1482,14 +1509,19 @@ def set_environment(self, key, value, relpath=False):
self.log.info("Not including statement to define environment variable $%s, as specified", key)
return ''

value, use_pushenv = self.unpack_setenv_value(key, value)

if relpath:
if value:
val = self.PATH_JOIN_TEMPLATE % value
else:
val = 'root'
else:
val = quote_str(value)
return 'setenv("%s", %s)\n' % (key, val)

env_setter = 'pushenv' if use_pushenv else 'setenv'

return '%s("%s", %s)\n' % (env_setter, key, val)

def swap_module(self, mod_name_out, mod_name_in, guarded=True):
"""
Expand Down
38 changes: 33 additions & 5 deletions test/framework/easyblock.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -33,6 +33,7 @@
import shutil
import sys
import tempfile
from distutils.version import LooseVersion
from inspect import cleandoc
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered, init_config
from unittest import TextTestRunner
Expand All@@ -48,7 +49,7 @@
from easybuild.tools.filetools import change_dir, copy_dir, copy_file, mkdir, read_file, remove_file
from easybuild.tools.filetools import verify_checksum, write_file
from easybuild.tools.module_generator import module_generator
from easybuild.tools.modules import reset_module_caches
from easybuild.tools.modules import EnvironmentModules, Lmod, reset_module_caches
from easybuild.tools.version import get_git_revision, this_is_easybuild
from easybuild.tools.py2vs3 import string_type

Expand DownExpand Up@@ -1158,8 +1159,20 @@ def test_make_module_step(self):
version = "3.14"
# purposely use a 'nasty' description, that includes (unbalanced) special chars: [, ], {, }
descr = "This {is a}} [fancy]] [[description]]. {{[[TEST}]"
modextravars = {'PI': '3.1415', 'FOO': 'bar'}
modextrapaths = {'PATH': ('bin', 'pibin'), 'CPATH': 'pi/include'}
modextravars = {
'PI': '3.1415',
'FOO': 'bar',
}

# also test setting environment variables via pushenv for Lmod or recent Environment Modules
is_modtool_ver51 = LooseVersion(self.modtool.version) >= LooseVersion('5.1')
if isinstance(self.modtool, Lmod) or (isinstance(self.modtool, EnvironmentModules) and is_modtool_ver51):
modextravars['TEST_PUSHENV'] = {'value': '123', 'pushenv': True}

modextrapaths = {
'PATH': ('xbin', 'pibin'),
'CPATH': 'pi/include',
}
self.contents = '\n'.join([
'easyblock = "ConfigureMake"',
'name = "%s"' % name,
Expand DownExpand Up@@ -1213,10 +1226,17 @@ def test_make_module_step(self):
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

for (key, val) in modextravars.items():
pushenv = False
if isinstance(val, dict):
pushenv = val.get('pushenv', False)
val = val['value']

env_setter = 'pushenv' if pushenv else 'setenv'

if get_module_syntax() == 'Tcl':
regex = re.compile(r'^setenv\s+%s\s+"%s"$' % (key, val), re.M)
regex = re.compile(r'^%s\s+%s\s+"%s"$' % (env_setter, key, val), re.M)
elif get_module_syntax() == 'Lua':
regex = re.compile(r'^setenv\("%s", "%s"\)$' % (key, val), re.M)
regex = re.compile(r'^%s\("%s", "%s"\)$' % (env_setter, key, val), re.M)
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
self.assertTrue(regex.search(txt), "Pattern %s found in %s" % (regex.pattern, txt))
Expand DownExpand Up@@ -1263,9 +1283,17 @@ def test_make_module_step(self):
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
self.assertFalse(regex.search(txt), "Pattern '%s' *not* found in %s" % (regex.pattern, txt))

os.environ['TEST_PUSHENV'] = '0'

# also check whether generated module can be loaded
self.modtool.load(['pi/3.14'])

# check value for $TEST_PUSHENV
if 'TEST_PUSHENV' in modextravars:
self.assertEqual(os.environ['TEST_PUSHENV'], '123')

self.modtool.unload(['pi/3.14'])
self.assertEqual(os.environ['TEST_PUSHENV'], '0')

# [==[ or ]==] in description is fatal
if get_module_syntax() == 'Lua':
Expand Down
2 changes: 1 addition & 1 deletion test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -69,7 +69,7 @@ def run(self, *args, **kwargs):
if self.cfg['toy_ext_param']:
run_cmd(self.cfg['toy_ext_param'])

return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper(), self.name)
return self.module_generator.set_environment('TOY_EXT_%s' % self.name.upper().replace('-', '_'), self.name)

def prerun(self):
"""
Expand Down