xref: /platform_testing/libraries/health/runners/longevity/platform/tests/src/android/platform/test/longevity/LongevityClassRunnerTest.java

/*
 * Copyright (C) 2019 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package android.platform.test.longevity;

import static com.google.common.truth.Truth.assertThat;

import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.argThat;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.MockitoAnnotations.initMocks;

import android.os.Bundle;
import android.platform.test.rule.TestWatcher;

import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.Description;
import org.junit.runner.RunWith;
import org.junit.runner.notification.Failure;
import org.junit.runner.notification.RunListener;
import org.junit.runner.notification.RunNotifier;
import org.junit.runners.JUnit4;
import org.junit.runners.model.FrameworkMethod;
import org.junit.runners.model.Statement;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.exceptions.base.MockitoAssertionError;

import java.util.ArrayList;
import java.util.List;

/** Unit tests for the {@link LongevityClassRunner}. */
public class LongevityClassRunnerTest {
    // Used to test things that are generated reflectively, e.g. dynamic rules.
    private static final List<String> sLogs = new ArrayList<>();

    public static class LoggingRule extends TestWatcher {
        private final String mName;

        public LoggingRule(String name) {
            mName = name;
        }

        @Override
        public void starting(Description description) {
            sLogs.add(String.format("%s starting", mName));
        }

        @Override
        public void finished(Description description) {
            sLogs.add(String.format("%s finished", mName));
        }
    }

    public static class InjectedRule1 extends LoggingRule {
        public InjectedRule1() {
            super("Injected rule 1");
        }
    }

    public static class InjectedRule2 extends LoggingRule {
        public InjectedRule2() {
            super("Injected rule 2");
        }
    }

    // We use two @Test methods here to distinguish between testing for class
    // rules vs. test rules. This is for testing only; we still advocate for
    // one @Test method per class.
    @RunWith(JUnit4.class)
    public static class LoggingTestWithRules {
        @ClassRule public static LoggingRule classRule = new LoggingRule("Hardcoded class rule");

        @Rule public LoggingRule testRule = new LoggingRule("Hardcoded test rule");

        @Test
        public void test1() {
            sLogs.add("Test 1 execution");
        }

        @Test
        public void test2() {
            sLogs.add("Test 2 execution");
        }
    }

    // A sample test class to test the runner with.
    @RunWith(JUnit4.class)
    public static class NoOpTest {
        @BeforeClass
        public static void beforeClassMethod() {}

        @Before
        public void beforeMethod() {}

        @Test
        public void testMethod() {}

        @After
        public void afterMethod() {}

        @AfterClass
        public static void afterClassMethod() {}
    }

    @RunWith(JUnit4.class)
    public static class FailingTest extends NoOpTest {
        @Test
        public void testMethod() {
            throw new RuntimeException("I failed.");
        }
    }

    @Mock private RunNotifier mRunNotifier;

    private LongevityClassRunner mRunner;

    private static final Statement PASSING_STATEMENT =
            new Statement() {
                public void evaluate() throws Throwable {
                    // No-op.
                }
            };

    private static final Statement FAILING_STATEMENT =
            new Statement() {
                public void evaluate() throws Throwable {
                    throw new RuntimeException("I failed.");
                }
            };

    // A failure message for assertion calls in Mockito stubs. These assertion failures will cause
    // the runner under test to fail but will not trigger a test failure directly. This message is
    // used to filter failures reported by the mocked RunNotifier and re-throw the ones injected in
    // the spy.
    private static final String ASSERTION_FAILURE_MESSAGE = "Test assertions failed";

    @Before
    public void setUp() {
        initMocks(this);
        sLogs.clear();
    }

    /**
     * Test that the {@link BeforeClass} methods are added to the test statement as {@link Before}
     * methods.
     */
    @Test
    public void testBeforeClassMethodsAddedAsBeforeMethods() throws Throwable {
        mRunner = spy(new LongevityClassRunner(NoOpTest.class));
        // Spy the withBeforeClasses() method to check that the method does not make changes to the
        // statement despite the presence of a @BeforeClass method.
        doAnswer(
                        invocation -> {
                            Statement returnedStatement = (Statement) invocation.callRealMethod();
                            // If this assertion fails, mRunNofitier will fire a test failure.
                            Assert.assertEquals(
                                    ASSERTION_FAILURE_MESSAGE,
                                    returnedStatement,
                                    invocation.getArgument(0));
                            return returnedStatement;
                        })
                .when(mRunner)
                .withBeforeClasses(any(Statement.class));
        // Spy the addRunBefores() method to check that the @BeforeClass method is added to the
        // @Before methods.
        doAnswer(
                        invocation -> {
                            List<FrameworkMethod> methodList =
                                    (List<FrameworkMethod>) invocation.getArgument(1);
                            // If any of these assertions fail, mRunNofitier will fire a test
                            // failure.
                            // There should be two methods.
                            Assert.assertEquals(ASSERTION_FAILURE_MESSAGE, methodList.size(), 2);
                            // The first one should be the @BeforeClass one.
                            Assert.assertEquals(
                                    ASSERTION_FAILURE_MESSAGE,
                                    methodList.get(0).getName(),
                                    "beforeClassMethod");
                            // The second one should be the @Before one.
                            Assert.assertEquals(
                                    ASSERTION_FAILURE_MESSAGE,
                                    methodList.get(1).getName(),
                                    "beforeMethod");
                            return invocation.callRealMethod();
                        })
                .when(mRunner)
                .addRunBefores(any(Statement.class), any(List.class), any(Object.class));
        // Run the runner.
        mRunner.run(mRunNotifier);
        verifyForAssertionFailures(mRunNotifier);
        // Verify that the stubbed methods are indeed called.
        verify(mRunner, times(1)).withBeforeClasses(any(Statement.class));
        verify(mRunner, times(1))
                .addRunBefores(any(Statement.class), any(List.class), any(Object.class));
    }

    /**
     * Test that the {@link AfterClass} methods are added to the test statement as potential {@link
     * After} methods.
     */
    @Test
    public void testAfterClassMethodsAddedAsAfterMethods() throws Throwable {
        mRunner = spy(new LongevityClassRunner(NoOpTest.class));
        // Spy the withAfterClasses() method to check that the method returns an instance of
        // LongevityClassRunner.RunAfterClassMethodsOnTestFailure.
        doAnswer(
                        invocation -> {
                            Statement returnedStatement = (Statement) invocation.callRealMethod();
                            // If this assertion fails, mRunNofitier will fire a test failure.
                            Assert.assertTrue(
                                    ASSERTION_FAILURE_MESSAGE,
                                    returnedStatement
                                            instanceof
                                            LongevityClassRunner.RunAfterClassMethodsOnTestFailure);
                            return returnedStatement;
                        })
                .when(mRunner)
                .withAfterClasses(any(Statement.class));
        // Spy the addRunAfters() method to check that the method returns an instance of
        // LongevityClassRunner.RunAfterMethods.
        doAnswer(
                        invocation -> {
                            Statement returnedStatement = (Statement) invocation.callRealMethod();
                            // If any of these assertions fail, mRunNotifier will fire a test
                            // failure.
                            Assert.assertTrue(
                                    ASSERTION_FAILURE_MESSAGE,
                                    returnedStatement
                                            instanceof LongevityClassRunner.RunAfterMethods);
                            // The second argument should only contain the @After method.
                            List<FrameworkMethod> afterMethodList =
                                    (List<FrameworkMethod>) invocation.getArgument(1);
                            Assert.assertEquals(
                                    ASSERTION_FAILURE_MESSAGE, afterMethodList.size(), 1);
                            Assert.assertEquals(
                                    ASSERTION_FAILURE_MESSAGE,
                                    afterMethodList.get(0).getName(),
                                    "afterMethod");
                            // The third argument should only contain the @AfterClass method.
                            List<FrameworkMethod> afterClassMethodList =
                                    (List<FrameworkMethod>) invocation.getArgument(2);
                            Assert.assertEquals(
                                    ASSERTION_FAILURE_MESSAGE, afterClassMethodList.size(), 1);
                            Assert.assertEquals(
                                    ASSERTION_FAILURE_MESSAGE,
                                    afterClassMethodList.get(0).getName(),
                                    "afterClassMethod");
                            return returnedStatement;
                        })
                .when(mRunner)
                .addRunAfters(
                        any(Statement.class), any(List.class), any(List.class), any(Object.class));
        // Run the runner.
        mRunner.run(mRunNotifier);
        verifyForAssertionFailures(mRunNotifier);
        // Verify that the stubbed methods are indeed called.
        verify(mRunner, times(1)).withAfterClasses(any(Statement.class));
        verify(mRunner, times(1))
                .addRunAfters(
                        any(Statement.class), any(List.class), any(List.class), any(Object.class));
    }

    /**
     * Test that {@link LongevityClassRunner.RunAfterMethods} marks the test as failed for a failed
     * test.
     */
    @Test
    public void testAfterClassMethodsHandling_marksFailure() throws Throwable {
        // Initialization parameter does not matter as this test does not concern the tested class.
        mRunner = spy(new LongevityClassRunner(NoOpTest.class));
        try {
            mRunner.hasTestFailed();
            Assert.fail("Test status should not be able to be checked before it's run.");
        } catch (Throwable e) {
            Assert.assertTrue(e.getMessage().contains("should not be checked"));
        }
        // Create and partially run the runner with a failing statement.
        Statement statement =
                mRunner.withAfters(
                        null,
                        new NoOpTest(),
                        new Statement() {
                            public void evaluate() throws Throwable {
                                throw new RuntimeException("I failed.");
                            }
                        });
        try {
            statement.evaluate();
        } catch (Throwable e) {
            // Expected and no action needed.
        }
        Assert.assertTrue(mRunner.hasTestFailed());
    }

    /**
     * Test that {@link LongevityClassRunner.RunAfterMethods} marks the test as passed for a passed
     * test.
     */
    @Test
    public void testRunAfterClassMethodsHandling_marksPassed() throws Throwable {
        // Initialization parameter does not matter as this test does not concern the tested class.
        mRunner = spy(new LongevityClassRunner(NoOpTest.class));
        // Checking test status before the statements are run should throw.
        try {
            mRunner.hasTestFailed();
            Assert.fail("Test status should not be able to be checked before it's run.");
        } catch (Throwable e) {
            Assert.assertTrue(e.getMessage().contains("should not be checked"));
        }
        // Create and partially run the runner with a passing statement.
        Statement statement =
                mRunner.withAfters(
                        null,
                        new NoOpTest(),
                        new Statement() {
                            public void evaluate() throws Throwable {
                                // Does nothing and thus passes.
                            }
                        });
        statement.evaluate();
        Assert.assertFalse(mRunner.hasTestFailed());
    }

    /** Test that {@link AfterClass} methods are run as {@link After} methods for a passing test. */
    @Test
    public void testAfterClass_runAsAfterForPassingTest() throws Throwable {
        // Initialization parameter does not matter as this test does not concern the tested class.
        mRunner = spy(new LongevityClassRunner(NoOpTest.class));
        // For a passing test, the AfterClass method should be run in the statement returned from
        // withAfters().
        Statement statement =
                mRunner.withAfters(
                        null, // Passing null as parameter in the interface is not actually used.
                        new NoOpTest(),
                        PASSING_STATEMENT);
        statement.evaluate();
        // Check that the @AfterClass method is called.
        ArgumentCaptor<List> methodsCaptor = ArgumentCaptor.forClass(List.class);
        verify(mRunner, times(1)).invokeAndCollectErrors(methodsCaptor.capture(), any());
        Assert.assertEquals(methodsCaptor.getValue().size(), 1);
        Assert.assertTrue(
                ((FrameworkMethod) methodsCaptor.getValue().get(0))
                        .getName()
                        .contains("afterClassMethod"));
    }

    /**
     * Test that {@link AfterClass} methods are run as {@link AfterClass} method for a failing test.
     */
    @Test
    public void testAfterClass_runAsAfterClassForFailingTest() throws Throwable {
        // Initialization parameter does not matter as this test does not concern the tested class.
        mRunner = spy(new LongevityClassRunner(NoOpTest.class));
        // For a failing test, the AfterClass method should be run in the statement returned from
        // withAfterClass().
        doReturn(true).when(mRunner).hasTestFailed();
        Statement statement = mRunner.withAfterClasses(FAILING_STATEMENT);
        try {
            statement.evaluate();
        } catch (Throwable e) {
            // Expected and no action needed.
        }
        // Check that the @AfterClass method is called.
        ArgumentCaptor<List> methodsCaptor = ArgumentCaptor.forClass(List.class);
        verify(mRunner, times(1)).invokeAndCollectErrors(methodsCaptor.capture(), any());
        Assert.assertEquals(methodsCaptor.getValue().size(), 1);
        Assert.assertTrue(
                ((FrameworkMethod) methodsCaptor.getValue().get(0))
                        .getName()
                        .contains("afterClassMethod"));
    }

    /** Test that {@link AfterClass} methods are only executed once for a passing test. */
    @Test
    public void testAfterClassRunOnlyOnce_passingTest() throws Throwable {
        mRunner = spy(new LongevityClassRunner(NoOpTest.class));
        mRunner.run(mRunNotifier);
        verify(mRunner, times(1))
                .invokeAndCollectErrors(getMethodNameMatcher("afterClassMethod"), any());
    }

    /** Test that {@link AfterClass} methods are only executed once for a failing test. */
    @Test
    public void testAfterClassRunOnlyOnce_failingTest() throws Throwable {
        mRunner = spy(new LongevityClassRunner(FailingTest.class));
        mRunner.run(mRunNotifier);
        verify(mRunner, times(1))
                .invokeAndCollectErrors(getMethodNameMatcher("afterClassMethod"), any());
    }

    /** Test that excluded classes are not executed. */
    @Test
    public void testIgnore_excludedClasses() throws Throwable {
        RunNotifier notifier = spy(new RunNotifier());
        RunListener listener = mock(RunListener.class);
        notifier.addListener(listener);
        Bundle ignores = new Bundle();
        ignores.putString(LongevityClassRunner.FILTER_OPTION, FailingTest.class.getCanonicalName());
        mRunner = spy(new LongevityClassRunner(FailingTest.class, ignores));
        mRunner.run(notifier);
        verify(listener, times(1)).testIgnored(any());
    }

    /** Test that the runner does not report iteration when iteration is not set. */
    @Test
    public void testReportIteration_noIterationSet() throws Throwable {
        ArgumentCaptor<Description> captor = ArgumentCaptor.forClass(Description.class);
        RunNotifier notifier = mock(RunNotifier.class);
        mRunner = spy(new LongevityClassRunner(NoOpTest.class));
        mRunner.run(notifier);
        verify(notifier).fireTestStarted(captor.capture());
        Assert.assertFalse(
                "Description class name should not contain the iteration number.",
                captor.getValue()
                        .getClassName()
                        .matches(
                                String.join(
                                        LongevityClassRunner.ITERATION_SEP_DEFAULT,
                                        "^.*",
                                        "[0-9]+$")));
    }

    /** Test that the runner reports iteration when set and the default separator was used. */
    @Test
    public void testReportIteration_withIteration_withDefaultSeparator() throws Throwable {
        ArgumentCaptor<Description> captor = ArgumentCaptor.forClass(Description.class);
        RunNotifier notifier = mock(RunNotifier.class);
        mRunner = spy(new LongevityClassRunner(NoOpTest.class));
        mRunner.setIteration(7);
        mRunner.run(notifier);
        verify(notifier).fireTestStarted(captor.capture());
        Assert.assertTrue(
                "Description class name should contain the iteration number.",
                captor.getValue()
                        .getClassName()
                        .matches(
                                String.join(
                                        LongevityClassRunner.ITERATION_SEP_DEFAULT, "^.*", "7$")));
    }

    /** Test that the runner reports iteration when set and a custom separator was supplied. */
    @Test
    public void testReportIteration_withIteration_withCustomSeparator() throws Throwable {
        String sep = "--";
        Bundle args = new Bundle();
        args.putString(LongevityClassRunner.ITERATION_SEP_OPTION, sep);

        ArgumentCaptor<Description> captor = ArgumentCaptor.forClass(Description.class);
        RunNotifier notifier = mock(RunNotifier.class);
        mRunner = spy(new LongevityClassRunner(NoOpTest.class, args));
        mRunner.setIteration(7);
        mRunner.run(notifier);
        verify(notifier).fireTestStarted(captor.capture());
        Assert.assertTrue(
                "Description class name should contain the iteration number.",
                captor.getValue().getClassName().matches(String.join(sep, "^.*", "7$")));
    }

    @Test
    public void testDynamicClassRules() throws Throwable {
        Bundle args = new Bundle();
        args.putString(
                LongevityClassRunner.DYNAMIC_OUTER_CLASS_RULES_OPTION,
                InjectedRule1.class.getName());
        args.putString(
                LongevityClassRunner.DYNAMIC_INNER_CLASS_RULES_OPTION,
                InjectedRule2.class.getName());

        RunNotifier notifier = mock(RunNotifier.class);
        mRunner = new LongevityClassRunner(LoggingTestWithRules.class, args);
        mRunner.run(notifier);
        verifyForAssertionFailures(notifier);
        assertThat(sLogs)
                .containsExactly(
                        "Injected rule 1 starting",
                        "Hardcoded class rule starting",
                        "Injected rule 2 starting",
                        "Hardcoded test rule starting",
                        "Test 1 execution",
                        "Hardcoded test rule finished",
                        "Hardcoded test rule starting",
                        "Test 2 execution",
                        "Hardcoded test rule finished",
                        "Injected rule 2 finished",
                        "Hardcoded class rule finished",
                        "Injected rule 1 finished");
    }

    @Test
    public void testDynamicTestRules() throws Throwable {
        Bundle args = new Bundle();
        args.putString(
                LongevityClassRunner.DYNAMIC_OUTER_TEST_RULES_OPTION,
                InjectedRule1.class.getName());
        args.putString(
                LongevityClassRunner.DYNAMIC_INNER_TEST_RULES_OPTION,
                InjectedRule2.class.getName());

        RunNotifier notifier = mock(RunNotifier.class);
        mRunner = new LongevityClassRunner(LoggingTestWithRules.class, args);
        mRunner.run(notifier);
        verifyForAssertionFailures(notifier);
        assertThat(sLogs)
                .containsExactly(
                        "Hardcoded class rule starting",
                        "Injected rule 1 starting",
                        "Hardcoded test rule starting",
                        "Injected rule 2 starting",
                        "Test 1 execution",
                        "Injected rule 2 finished",
                        "Hardcoded test rule finished",
                        "Injected rule 1 finished",
                        "Injected rule 1 starting",
                        "Hardcoded test rule starting",
                        "Injected rule 2 starting",
                        "Test 2 execution",
                        "Injected rule 2 finished",
                        "Hardcoded test rule finished",
                        "Injected rule 1 finished",
                        "Hardcoded class rule finished");
    }

    private List<FrameworkMethod> getMethodNameMatcher(String methodName) {
        return argThat(
                l ->
                        l.stream()
                                .anyMatch(
                                        f -> ((FrameworkMethod) f).getName().contains(methodName)));
    }

    /**
     * Verify that no test failure is fired because of an assertion failure in the stubbed methods.
     * If the verfication fails, check whether it's due the injected assertions failing. If yes,
     * throw that exception out; otherwise, throw the first exception.
     */
    private void verifyForAssertionFailures(final RunNotifier notifier) throws Throwable {
        try {
            verify(notifier, never()).fireTestFailure(any());
        } catch (MockitoAssertionError e) {
            ArgumentCaptor<Failure> failureCaptor = ArgumentCaptor.forClass(Failure.class);
            verify(notifier, atLeastOnce()).fireTestFailure(failureCaptor.capture());
            List<Failure> failures = failureCaptor.getAllValues();
            // Go through the failures, look for an known failure case from the above exceptions
            // and throw the exception in the first one out if any.
            for (Failure failure : failures) {
                if (failure.getException().getMessage().contains(ASSERTION_FAILURE_MESSAGE)) {
                    throw failure.getException();
                }
            }
            // Otherwise, throw the exception from the first failure reported.
            throw failures.get(0).getException();
        }
    }
}