I use this script to do regression testing:
# Display regression script.
# Usage:
# Set action to "setup" and run this script in the reference
# copy of Rhino. This will generate a series of "* - reference.png"
# files.
#
# Then, set action to "test" and run in your target copy of Rhino.
#
# At the end you will see "x of y passed tests" plus a list of all tests that failed.
#
# All resulting images are in the "Results" folder.
#
# To add test cases, add:
# * 3DM files to the "Objects" folder. Objects in the model
# that should have materials asigned must be named "material_target"
# * DisplayMode.ini files to the "DisplayModes" folder
# * Material.rmtl files to the "Materials" folder
#
# IMPORTANT!!!
# Make sure that ALL built-in display modes have been set to their defaults
# in both situations (i.e.) Confirm they're at their defaults before running
# "Setup" phase, and confirm they're at their defaults before running the
# "Test" phase. Since setup and test can be run at different points in time
# and even on different configurations, it is impotant that you make sure
# you're running regression tests using the EXACT same display settings...
# otherwise, it's possible that some, or even all, tests will fail
#
action = 'setup' # To be run in known "good" instance of Rhino.
#action = 'test' # To be run in the Rhino to test.
#action = 'db' # Setup or update the test database.
samples = 100
# match_amount controls how much the test frames must match the reference frames
# This number is the number of different pixels divided by the total number
# of pixels in the frame. When all are identical, this number is 1.000. When
# all are different, this number is 0.000. Frames must match more than this much
# for the tests to pass:
match_amount = 0.999
standard_res = (900,450)
high_res = (standard_res[0]*4, standard_res[1]*4)
import clr
clr.AddReference("System.Xml.Linq")
import System.Xml.Linq as xml
import os, math
import Rhino
import System.Drawing
import time
import scriptcontext as sc
import itertools
import collections
import hashlib
import webbrowser
from Rhino.Display import ViewCapture
this_dir = str(os.path.dirname(os.path.realpath(__file__)))
model_dir = str(os.path.join(this_dir, "Objects"))
output_dir = str(os.path.join(this_dir, "Results"))
bgstyle = sc.doc.RenderSettings.BackgroundStyle # SolidColor, Gradient, Environment
programContext = Rhino.Render.RenderContent.ChangeContexts.Program
colEnvs = ["color1.renv", "color2.renv", "color3.renv"]
hdrEnvs = ["hdr1.renv", "hdr2.renv", "hdr3.renv"]
planarEnvs = ["planar1.renv", "planar2.renv", "planar3.renv"]
gpChoices = [
(True, "shadows-only"),
(True, "material"),
(False, "disabled")]
envBgChoices = [
(bgstyle.Environment, hdrEnvs[0]),
(bgstyle.Environment, planarEnvs[0]),
(bgstyle.Environment, colEnvs[0]),
(bgstyle.Gradient, "gradient"),
(bgstyle.SolidColor, "solid")]
custReflectChoices = [
(True, hdrEnvs[1]),
(True, planarEnvs[1]),
(True, colEnvs[1]),
(False, "no-custom-reflection")]
skylightChoices = [
(False,"no-skylight"),
(True, hdrEnvs[1]),
(True, hdrEnvs[2]),
(True, planarEnvs[1]),
(True, planarEnvs[2]),
(True, colEnvs[1]),
(True, colEnvs[2]),
(True, "no-custom-skylight")]
sunChoices = [(True, "sun"), (False, "no-sun")]
possibleWorlds = [
x
for x
in itertools.product(
gpChoices,
envBgChoices,
custReflectChoices,
skylightChoices,
sunChoices)
]
possibleWorldsSha1 = hashlib.sha1()
possibleWorldsSha1.update("{0}".format(possibleWorlds))
possibleWorldsName = possibleWorldsSha1.hexdigest()
def irange(a, b):
"""
Range from a to b inclusive.
"""
return range(a, b+1)
def flatten(l):
"""
flatten a list that can contain nested lists.
"""
for el in l:
if isinstance(el, collections.Iterable) and not isinstance(el, basestring):
for sub in flatten(el):
yield sub
else:
yield el
def test_ids(l, max):
"""
Generate index list from test ids. Can be used to generate broken sequences.
ranges tests_to_run = test_ids([irange(1,3), irange(6,10)], len(possibleWorlds))
will yield [1,2,3,6,7,8,9,10]
"""
ids = [x-1 for x in flatten(l) if (x-1)>=0 and x<=max]
ids = collections.OrderedDict((x, True) for x in ids).keys()
return ids
def colordiff(colora, colorb):
a = Rhino.Display.ColorLAB(colora)
b = Rhino.Display.ColorLAB(colorb)
lsq = (a.L-b.L)**2
asq = (a.A-b.A)**2
bsq = (a.B-b.B)**2
delta_e = math.sqrt(lsq+asq+bsq)
return delta_e
def compare_images(a, b):
'''Compare two System.Drawing.Bitmap images'''
pixel_count = 0
identical_pixel_count = 0
errorbmp = System.Drawing.Bitmap(b.Width, b.Height)
for x in range(0, b.Width):
for y in range(0, b.Height):
pixel_count += 1
a_color = a.GetPixel(x,y)
b_color = b.GetPixel(x,y)
if a_color == b_color:
identical_pixel_count += 1
errorbmp.SetPixel(x,y, System.Drawing.Color.FromArgb(0,0,0))
else:
delta_e = colordiff(a_color, b_color)
pcol = int(255.0 * delta_e)
errorbmp.SetPixel(x,y, System.Drawing.Color.FromArgb(pcol, pcol, pcol))
if(delta_e < 0.6):
identical_pixel_count += 1
# Compute percent difference between two images
return (float(identical_pixel_count) / float(pixel_count), errorbmp)
def capture_viewport(display_mode, resolution):
view = sc.doc.Views.ActiveView
if display_mode:
Rhino.RhinoApp.RunScript("-_SetDisplayMode _Mode={0}".format(display_mode), False)
else:
Rhino.RhinoApp.RunScript("-_line 0,0 1,1 sellast delete", False)
view.Redraw()
vc = ViewCapture()
vc.Width = resolution[0]
vc.Height = resolution[1]
vc.RealtimeRenderPasses = samples
return vc.CaptureToBitmap(view)
def get_renv(name):
name = name.replace(".renv", "")
doc = sc.doc
for e in doc.RenderEnvironments:
if e.Name == name: return e
return None
def xn(s):
"""
Shorthand for xml.XName.Get(s).
"""
return xml.XName.Get(s)
def xel(*args):
"""
Shorthand for xml.XElement(*args).
"""
return xml.XElement(*args)
true = "true"
false = "false"
def cleanpath(p):
if os.path.exists(p):
os.unlink(p)
class TestCase:
"""
TestCase represents a collection of settings that constitute one
test case (or setup thereof).
"""
def __init__(self, id, model_path, action, display_modes, world, resolution):
"""
Initialize a new TestCase.
id (int) - is a numerical rank index.
model_path(str) - the path to the model to use for testing
action (str) - either 'test' or 'setup'. Setup creates captures, test creates several captures and compares them with
captures created in setup.
display_mode (str) - a string representing a display mode to test with
world (tuple) - a tuple containing settings for the world
resolution (tuple) - a tuple containing the resolution at which to create the capture at.
"""
self.id = id
self.model_path = model_path
self.action = action
self.display_modes = display_modes
self.world = world
self.resolution = resolution
sh = hashlib.sha1()
_fnm = os.path.splitext(os.path.basename(self.model_path))[0]
sh.update("{0}".format((_fnm, self.world)))
self.name = sh.hexdigest()
self.reference_image_name = dict()
self.test1_image_name = dict()
self.test1_error_name = dict()
self.test2_image_name = dict()
self.test2_error_name = dict()
self.timings = dict()
self.reference_compares = dict()
for dm in self.display_modes:
self.reference_image_name[dm] = "{0:06d}_{1}_{2}_{3}x{4}_reference.png".format(self.id, self.name, dm, self.resolution[0], self.resolution[1])
self.test1_image_name[dm] = "{0:06d}_{1}_{2}_{3}x{4}_test1.png".format(self.id, self.name, dm, self.resolution[0], self.resolution[1])
self.test1_error_name[dm] = "{0:06d}_{1}_{2}_{3}x{4}_error.png".format(self.id, self.name, dm, self.resolution[0], self.resolution[1])
self.test2_image_name[dm] = "{0:06d}_{1}_{2}_{3}x{4}_test2.png".format(self.id, self.name, dm, self.resolution[0], self.resolution[1])
self.test2_error_name[dm] = "{0:06d}_{1}_{2}_{3}x{4}_error.png".format(self.id, self.name, dm, self.resolution[0], self.resolution[1])
self.timings[dm] = -1.0
if len(self.display_modes)>1:
combos = list(itertools.combinations(self.display_modes, 2))
for combo in combos:
pass
self.bakefile_name = "{0:06d}_{1}.3dm".format(self.id, self.name)
self.cf = -1.0
def prepare_model(self):
"""
Load the model from filename. Environments (.renv) will be imported. The
model will be changed according the settings given in world.
The image name for the model will be returned.
"""
doc = sc.doc
doc.Open(self.model_path)
doc = sc.doc
view = doc.Views.ActiveView
if self.world:
envdir = os.path.dirname(self.model_path)
# import test environments
for f in os.listdir(envdir):
if not f.lower().endswith(".renv"): continue
envimp = os.path.join(envdir, f)
Rhino.RhinoApp.RunScript("""-_Environments
Options
LoadFromFile
"{0}"
Enter
Enter
""".format(envimp), False)
doc.UndoRecordingEnabled = False
# set up ground plane
gpsettings = self.gp_settings()
gp = doc.GroundPlane
gp.BeginChange(programContext)
gp.Enabled = gpsettings[0]
gp.ShadowOnly = gpsettings[1]=="shadows-only"
gp.EndChange()
# set bg
bgsettings = self.bg_settings()
rendersettings = doc.RenderSettings
rendersettings.BackgroundStyle = bgsettings[0]
if bgsettings[0] == bgstyle.Environment:
#print("setting environment to", bgsettings[1])
doc.CurrentEnvironment.ForBackground = get_renv(bgsettings[1])
else:
doc.CurrentEnvironment.ForBackground = None
# set custom reflection
reflsettings = self.refl_settings()
if reflsettings[0]:
doc.CurrentEnvironment.ForReflectionAndRefraction = get_renv(reflsettings[1])
else:
doc.CurrentEnvironment.ForReflectionAndRefraction = None
# set skylight
skysettings = self.sky_settings()
skylight = doc.Lights.Skylight
skylight.BeginChange(programContext)
skylight.Enabled = skysettings[0]
skylight.EndChange()
if skysettings[0]:
skyenv = get_renv(skysettings[1])
print("setting sky env {0} [{1}]".format(skysettings[0], skyenv))
doc.CurrentEnvironment.ForLighting = get_renv(skysettings[1])
else:
print("no sky")
doc.CurrentEnvironment.ForLighting = None
# set sun
sunsettings = self.sun_settings()
sun = doc.Lights.Sun
sun.BeginChange(programContext)
sun.Enabled = sunsettings[0]
if sun.Enabled:
sun.SetPosition(120, 45)
sun.EndChange()
Rhino.RhinoApp.RunScript("-_line 0,0 1,1 sellast delete", False)
view.Redraw()
def gp_settings(self):
"""
Get the groundplane settings from the world settings.
"""
return self.world[0]
def bg_settings(self):
"""
Get the background (render) settings from the world settings.
"""
return self.world[1]
def refl_settings(self):
"""
Get the custom reflection environment settings from the world settings.
"""
return self.world[2]
def sky_settings(self):
"""
Get the skylight settings from the world settings.
"""
return self.world[3]
def sun_settings(self):
"""
Get the sun settings from the world settings.
"""
return self.world[4]
def gp_xml(self):
"""
Create ground plane XML representation for database.
"""
gps = self.gp_settings()
g = xel(xn("groundplane"),
xel("enabled", gps[0]),
xel("shadows-only", gps[1]=="shadows-only")
)
# g.SetAttributeValue(xn("enabled"), gps[0])
# g.SetAttributeValue(xn("shadows-only"), gps[1]=="shadows-only")
return g
def refl_settings_xml(self):
"""
Create custom reflection environment XML representation for database.
"""
refl = self.refl_settings()
r = xel(xn("custom_reflection"),
xel(xn("use_custom"), refl[0]),
xel(xn("reflenv"), refl[1])
)
return r
def render_settings_xml(self):
"""
Create render settings XML representation for database.
"""
rs = self.bg_settings()
r = xel(xn("rendersettings"),
xel(xn("backgroundstyle"), rs[0]),
xel(xn("bgenv"), rs[1])
)
return r
def sky_settings_xml(self):
"""
Create sky settings XML representation for database.
"""
ss = self.sky_settings()
s = xel(xn("skylight"),
xel(xn("enabled"), ss[0]),
xel(xn("skyenv"), ss[1])
)
return s
def sun_settings_xml(self):
"""
Create sun settings XML representation for database.
"""
ss = self.sun_settings()
s = xel(xn("sun"),
xel(xn("enabled"), ss[0])
)
return s
def reference_files(self):
fs = [xel(xn("file"), xel(xn("display_mode"), dm), xel(xn("path"), self.reference_image_name[dm])) for dm in self.reference_image_name]
rf = xel(xn("reference_files"), fs)
return rf
def xml(self):
t = xel(xn("case")
, self.gp_xml()
, self.render_settings_xml()
, self.refl_settings_xml()
, self.sky_settings_xml()
, self.sun_settings_xml()
, self.reference_files()
)
t.SetAttributeValue(xn("id"), self.id)
t.SetAttributeValue(xn("code"), self.name)
return t
def test_single(self, display_mode):
try:
reference_image = System.Drawing.Bitmap(os.path.join(output_dir, self.reference_image_name[display_mode]))
except:
return 0
Rhino.RhinoApp.RunScript("-_SetDisplayMode _Mode={0}".format("Wireframe"), False)
Rhino.RhinoApp.RunScript("RotateView Left RotateView Right", False)
time1 = time.time()
image1 = capture_viewport(display_mode, self.resolution)
time1 = time.time() - time1
image1path = os.path.join(output_dir, self.test1_image_name[display_mode])
cleanpath(image1path)
image1.Save(image1path, System.Drawing.Imaging.ImageFormat.Png)
Rhino.RhinoApp.RunScript("-_SetDisplayMode _Mode={0}".format("Wireframe"), False)
Rhino.RhinoApp.RunScript("RotateView Left RotateView Right", False)
time2 = time.time()
image3 = capture_viewport(display_mode, self.resolution)
time2 = time.time() - time2
image3path = os.path.join(output_dir, self.test2_image_name[display_mode])
cleanpath(image3path)
image3.Save(image3path, System.Drawing.Imaging.ImageFormat.Png)
f1_compare, err1 = compare_images(reference_image, image1)
f3_compare, err2 = compare_images(reference_image, image3)
err1path = os.path.join(output_dir, self.test1_error_name[display_mode])
err2path = os.path.join(output_dir, self.test2_error_name[display_mode])
cleanpath(err1path)
cleanpath(err2path)
err1.Save(err1path, System.Drawing.Imaging.ImageFormat.Png)
err2.Save(err2path, System.Drawing.Imaging.ImageFormat.Png)
image1.Dispose()
image3.Dispose()
err1.Dispose()
err2.Dispose()
self.timings[display_mode] = (time1 + time2) / 2.0
sc.doc.Modified = False
return (f1_compare + f3_compare) / 2.0
def open3dm(self):
doc = sc.doc
fullpathbaked = os.path.join(this_dir, "Baked", possibleWorldsName, self.bakefile_name)
doc.Open(fullpathbaked)
print("opened", fullpathbaked)
def setup_single(self, display_mode):
"""
Load given model from model_path, creating a capture in display_mode.
The model is prepared with settings from world, and captured at resolution.
Returns the full path to the image rendered.
"""
fullpath = ""
#self.open3dm()
#self.prepare_model()
Rhino.RhinoApp.RunScript("-_SetDisplayMode _Mode={0}".format("Wireframe"), False)
#Rhino.RhinoApp.RunScript("RotateView Left RotateView Right", False)
start = time.time()
image = capture_viewport(display_mode, self.resolution)
self.timings[display_mode] = time.time() - start
fullpath = os.path.join(output_dir, self.reference_image_name[display_mode])
cleanpath(fullpath)
print(fullpath)
image.Save(fullpath, System.Drawing.Imaging.ImageFormat.Png)
sc.doc.Modified = False
#return fullpath
def create_3dm(self, root):
"""
Load given model from model_path, setting it up according the
settings specified in world. The resulting model will be saved
to the Baked folder for future reference.
Returns the full path to the created file.
"""
fullpath = ""
self.prepare_model()
Rhino.RhinoApp.RunScript("-_line 0,0 1,1 sellast delete", False)
Rhino.RhinoApp.RunScript("RotateView Left RotateView Right", False)
fwo = Rhino.FileIO.FileWriteOptions()
fwo.IncludeBitmapTable = True
fwo.IncludeRenderMeshes = True
fwo.SuppressDialogBoxes = True
fullpath = os.path.join(root, "Baked", possibleWorldsName, self.bakefile_name)
cleanpath(fullpath)
fullpathbak = fullpath+"bak"
dirname = os.path.dirname(fullpath)
if not os.path.exists(dirname):
os.makedirs(dirname)
sc.doc.WriteFile(fullpath, fwo)
sc.doc.Modified = False
cleanpath(fullpathbak)
return fullpath
def html(self):
info = ""
for dm in self.display_modes:
info += r"""
<td>
<h3>{0:06d}: {1} : {2} seconds</h3>
</td>
""".format(self.id, dm, self.timings[dm])
dmres = ""
for dm in self.display_modes:
dmres += r"""
<td>
<a href="{0}"><img src="{0}" width="300" border="0" title="{1:06d}:{2} - {3} seconds"/></a>
</td>
""".format(self.reference_image_name[dm], self.id, dm, self.timings[dm])
templ = r"""
<table border="1">
<tr>
{0}
</tr>
<tr colspan="2">
<a href="{3}">{1}</a>
</tr>
<tr>
{2}
</tr>
</table>
""".format(info, self.world, dmres, self.bakefile_name)
return templ
def do(self):
"""
Execute the action specified.
"""
sc.doc.Modified = False
if self.action == 'setup':
self.create_3dm(this_dir)
sc.doc.Modified = False
#self.prepare_model()
self.open3dm()
sc.doc.Modified = False
for dm in self.display_modes:
if self.action == 'test':
self.cf = self.test_single(dm)
#if self.cf < match_amount:
# failed.append(model_path + ": {0}".format(cf))
#else:
# passed.append(model_path)
else:
self.setup_single(dm)
sc.doc.Modified = False
def suite(all_cases):
s = xel(xn("suite"), [t.xml() for t in all_cases])
return s
def create_database(all_cases):
s = suite(all_cases)
print(s)
return s
def run(action):
display_modes = ['Raytraced', 'Rendered']
#display_modes = ['Raytraced']
start_all = time.time()
all_cases = []
run_cases = []
worldcount = len(possibleWorlds)
# all tests: tests_to_run = test_ids([irange(1,len(possibleWorlds))], len(possibleWorlds))
tests_to_run = test_ids([irange(1,worldcount)], worldcount)
# how to do different ranges tests_to_run = test_ids([irange(1,3), irange(6,10)], len(possibleWorlds))
#tests_to_run = test_ids([irange(101, 200)], len(possibleWorlds))
#tests_to_run = test_ids([irange(1,10)], len(possibleWorlds))
#tests_to_run = test_ids([irange(worldcount-3,worldcount)], worldcount)
tests_to_run = test_ids([129], worldcount)
print(tests_to_run)
for ttr in tests_to_run:
print(possibleWorlds[ttr])
# Throw away modifications in current document
sc.doc.Modified = False
root = os.path.join(this_dir, "Environments")
for file in os.listdir(root):
if not file.lower().endswith(".3dm"):
continue
model_path = os.path.join(root, file)
all_cases.extend([TestCase(i+1, model_path, action, display_modes, world, standard_res) for i, world in enumerate(possibleWorlds)])
for ttr in tests_to_run:
print(all_cases[ttr].world, all_cases[ttr].bakefile_name)
#if action == 'setup':
# for file in os.listdir(output_dir):
# os.unlink(os.path.join(output_dir, file))
if action == 'db':
create_database(all_cases)
else:
root = os.path.join(this_dir, "Environments")
cnt = 0
for file in os.listdir(root):
if not file.lower().endswith(".3dm"):
continue
model_path = os.path.join(root, file)
for t in tests_to_run:
#world = possibleWorlds[t]
cnt = cnt + 1
#if cnt > 2: break
test_case = all_cases[t]# TestCase(cnt, model_path, action, display_modes, world, standard_res)
test_case.do()
run_cases.append(test_case)
total_time = time.time() - start_all
resultshtml = os.path.join(output_dir, "results.html")
with open(resultshtml, "w") as f:
f.write("<html><body>")
f.write("<p>{0} tests ({1} renders) completed in {2} seconds</p>".format(len(run_cases), len(run_cases) * len(display_modes), total_time))
for tc in all_cases:
f.write(tc.html())
f.write("</body></html>")
webbrowser.open_new_tab(resultshtml)
runresultshtml = os.path.join(output_dir, "runresults.html")
with open(runresultshtml, "w") as f:
f.write("<html><body>")
f.write("<p>{0} tests ({1} renders) completed in {2} seconds</p>".format(len(run_cases), len(run_cases) * len(display_modes), total_time))
for tc in run_cases:
f.write(tc.html())
f.write("</body></html>")
webbrowser.open_new_tab(runresultshtml)
# Throw away modifications in current document, in case some were left
sc.doc.Modified = False
if __name__ == '__main__':
if not os.path.exists(output_dir):
os.mkdir(output_dir)
run(action)